--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:38:04.949138Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4547e-65765f9c8-v4n9c namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 20949593-c6e2-4226-b094-727a339e479b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-4547e-65765f9c8-v4n9c to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:38:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:38:04Z" name: ensemble-graph-4547e-65765f9c8-v4n9c.18a8b4a79ae8bfd7 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: 306e337e-c35f-4e36-9cca-e295ec4797ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:05Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4547e-65765f9c8-v4n9c namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 20949593-c6e2-4226-b094-727a339e479b kind: Event lastTimestamp: "2026-04-22T14:38:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-4547e-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T14:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:05Z" name: ensemble-graph-4547e-65765f9c8-v4n9c.18a8b4a7a93ccafb namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: 23663697-f3bb-4126-8cca-097e9b194a7c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:05Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4547e-65765f9c8-v4n9c namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: 20949593-c6e2-4226-b094-727a339e479b kind: Event lastTimestamp: "2026-04-22T14:38:05Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:38:05Z" name: ensemble-graph-4547e-65765f9c8-v4n9c.18a8b4a7d7163274 namespace: kserve-ci-e2e-test resourceVersion: "20588" uid: 39d00605-e721-4231-b879-73e3a74dbef2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4547e} kind: Pod name: ensemble-graph-4547e-65765f9c8-v4n9c namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 20949593-c6e2-4226-b094-727a339e479b kind: Event lastTimestamp: "2026-04-22T14:38:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:05Z" name: ensemble-graph-4547e-65765f9c8-v4n9c.18a8b4a7d88f37c3 namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 5a10c92b-a1e6-42e8-a537-ea6a420ecc79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4547e} kind: Pod name: ensemble-graph-4547e-65765f9c8-v4n9c namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 20949593-c6e2-4226-b094-727a339e479b kind: Event lastTimestamp: "2026-04-22T14:38:06Z" message: 'Created container: ensemble-graph-4547e' metadata: creationTimestamp: "2026-04-22T14:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:06Z" name: ensemble-graph-4547e-65765f9c8-v4n9c.18a8b4a7db97351f namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: dfcac193-e4b7-4b23-b4b9-4b73db71a17a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4547e} kind: Pod name: ensemble-graph-4547e-65765f9c8-v4n9c namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 20949593-c6e2-4226-b094-727a339e479b kind: Event lastTimestamp: "2026-04-22T14:38:06Z" message: Started container ensemble-graph-4547e metadata: creationTimestamp: "2026-04-22T14:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:06Z" name: ensemble-graph-4547e-65765f9c8-v4n9c.18a8b4a7dc05d38d namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: d56c2854-d0d5-490b-b2e2-a1655f030072 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4547e} kind: Pod name: ensemble-graph-4547e-65765f9c8-v4n9c namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 20949593-c6e2-4226-b094-727a339e479b kind: Event lastTimestamp: "2026-04-22T14:46:19Z" message: Stopping container ensemble-graph-4547e metadata: creationTimestamp: "2026-04-22T14:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:19Z" name: ensemble-graph-4547e-65765f9c8-v4n9c.18a8b51ad1ad2998 namespace: kserve-ci-e2e-test resourceVersion: "24086" uid: 6585a741-2719-4913-b2e2-ba9dc444dd76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:46:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4547e} kind: Pod name: ensemble-graph-4547e-65765f9c8-v4n9c namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 20949593-c6e2-4226-b094-727a339e479b kind: Event lastTimestamp: "2026-04-22T14:46:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:47Z" name: ensemble-graph-4547e-65765f9c8-v4n9c.18a8b51b7d6ac68a namespace: kserve-ci-e2e-test resourceVersion: "24399" uid: 8ba6ad12-7a77-41a1-bbf8-0b8115a41e36 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-4547e-65765f9c8 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: 2cb4e233-ddae-4b7d-b465-670208f07f9a kind: Event lastTimestamp: "2026-04-22T14:38:04Z" message: 'Created pod: ensemble-graph-4547e-65765f9c8-v4n9c' metadata: creationTimestamp: "2026-04-22T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:04Z" name: ensemble-graph-4547e-65765f9c8.18a8b4a79a55d993 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 364ee471-1a83-4fc3-bfd2-bd3561acc0a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-4547e namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: c5976cfa-066f-4aa6-aa0e-0bf7949f780a kind: Event lastTimestamp: "2026-04-22T14:38:04Z" message: Scaled up replica set ensemble-graph-4547e-65765f9c8 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:04Z" name: ensemble-graph-4547e.18a8b4a799805e1d namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 95ea44cf-a13b-4f1c-850a-e8490d6663e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4547e namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 078474d2-47b4-4c85-a274-f0f26e9baaab kind: Event lastTimestamp: "2026-04-22T14:38:04Z" message: InferenceGraph [ensemble-graph-4547e] is Ready metadata: creationTimestamp: "2026-04-22T14:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:38:05Z" name: ensemble-graph-4547e.18a8b4a79dea9844 namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 6aee9fcf-0a02-4101-b664-e960afca7492 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:38:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4547e namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 3e5bd966-079d-4fd8-aae3-e6b9b56c5c04 kind: Event lastTimestamp: "2026-04-22T14:38:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:34Z" name: ensemble-graph-4547e.18a8b4ab18065d3a namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: b9f7f2e4-2ad7-4b23-860f-3b4e1631ef90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:38:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4547e namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 3e5bd966-079d-4fd8-aae3-e6b9b56c5c04 kind: Event lastTimestamp: "2026-04-22T14:38:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:34Z" name: ensemble-graph-4547e.18a8b4ab183f5c4c namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 25a1541f-afaa-48aa-8c5c-1a82abc45d20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:36:44.650821Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-dee6f-547898ffd6-nngnb namespace: kserve-ci-e2e-test resourceVersion: "19540" uid: 8685743d-a69d-4c41-88b8-2459f83740a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-dee6f-547898ffd6-nngnb to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:36:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:36:44Z" name: ensemble-graph-dee6f-547898ffd6-nngnb.18a8b494e8c1e4db namespace: kserve-ci-e2e-test resourceVersion: "19545" uid: 04953124-88fe-43f0-aa23-b249388d9510 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:44Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-dee6f-547898ffd6-nngnb namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 8685743d-a69d-4c41-88b8-2459f83740a4 kind: Event lastTimestamp: "2026-04-22T14:36:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-dee6f-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T14:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:44Z" name: ensemble-graph-dee6f-547898ffd6-nngnb.18a8b494f74e8c6f namespace: kserve-ci-e2e-test resourceVersion: "19559" uid: 62875fe7-fa3a-4b27-b0f7-848a3830cfe7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:45Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-dee6f-547898ffd6-nngnb namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: 8685743d-a69d-4c41-88b8-2459f83740a4 kind: Event lastTimestamp: "2026-04-22T14:36:45Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:36:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:36:45Z" name: ensemble-graph-dee6f-547898ffd6-nngnb.18a8b49524f12ef2 namespace: kserve-ci-e2e-test resourceVersion: "19569" uid: 7707faf7-5ba4-4948-8023-f9bbe8cb114e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dee6f} kind: Pod name: ensemble-graph-dee6f-547898ffd6-nngnb namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 8685743d-a69d-4c41-88b8-2459f83740a4 kind: Event lastTimestamp: "2026-04-22T14:36:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:36:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:45Z" name: ensemble-graph-dee6f-547898ffd6-nngnb.18a8b495263b8f5c namespace: kserve-ci-e2e-test resourceVersion: "19571" uid: df22ac5f-8f26-4964-b9b6-38821bb2ca0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dee6f} kind: Pod name: ensemble-graph-dee6f-547898ffd6-nngnb namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 8685743d-a69d-4c41-88b8-2459f83740a4 kind: Event lastTimestamp: "2026-04-22T14:36:45Z" message: 'Created container: ensemble-graph-dee6f' metadata: creationTimestamp: "2026-04-22T14:36:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:45Z" name: ensemble-graph-dee6f-547898ffd6-nngnb.18a8b495291262af namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: b3a8d57d-ccf4-4360-979f-4475244d4413 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dee6f} kind: Pod name: ensemble-graph-dee6f-547898ffd6-nngnb namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 8685743d-a69d-4c41-88b8-2459f83740a4 kind: Event lastTimestamp: "2026-04-22T14:36:45Z" message: Started container ensemble-graph-dee6f metadata: creationTimestamp: "2026-04-22T14:36:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:45Z" name: ensemble-graph-dee6f-547898ffd6-nngnb.18a8b49529dbce84 namespace: kserve-ci-e2e-test resourceVersion: "19573" uid: bd79d08b-52c9-47ea-9f88-bea62c91ff50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dee6f} kind: Pod name: ensemble-graph-dee6f-547898ffd6-nngnb namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 8685743d-a69d-4c41-88b8-2459f83740a4 kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: Stopping container ensemble-graph-dee6f metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:54Z" name: ensemble-graph-dee6f-547898ffd6-nngnb.18a8b49741ffe0eb namespace: kserve-ci-e2e-test resourceVersion: "19661" uid: 53583a7c-812b-4df7-b955-f57e9b40d287 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:36:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-dee6f} kind: Pod name: ensemble-graph-dee6f-547898ffd6-nngnb namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 8685743d-a69d-4c41-88b8-2459f83740a4 kind: Event lastTimestamp: "2026-04-22T14:37:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:36:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:22Z" name: ensemble-graph-dee6f-547898ffd6-nngnb.18a8b497e22c3dce namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 2d9cca7c-2a2a-475d-ad9b-10a12fc812b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-dee6f-547898ffd6 namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: 1a0d4e8b-57dc-4bca-b311-86d50247de57 kind: Event lastTimestamp: "2026-04-22T14:36:44Z" message: 'Created pod: ensemble-graph-dee6f-547898ffd6-nngnb' metadata: creationTimestamp: "2026-04-22T14:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:44Z" name: ensemble-graph-dee6f-547898ffd6.18a8b494e857352d namespace: kserve-ci-e2e-test resourceVersion: "19541" uid: aa7fa180-be9a-45cc-a48e-ef66b52a08ce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-dee6f namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: 21af00e8-3eb1-449f-b9c6-8dcf929b080f kind: Event lastTimestamp: "2026-04-22T14:36:44Z" message: Scaled up replica set ensemble-graph-dee6f-547898ffd6 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:44Z" name: ensemble-graph-dee6f.18a8b494e78c4e7a namespace: kserve-ci-e2e-test resourceVersion: "19538" uid: 6d54dc0d-98e3-42cb-8c42-1ad5a125e471 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-dee6f namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: 179f112b-4e03-40d5-94d3-d0e2eb842db9 kind: Event lastTimestamp: "2026-04-22T14:36:44Z" message: InferenceGraph [ensemble-graph-dee6f] is Ready metadata: creationTimestamp: "2026-04-22T14:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:44Z" name: ensemble-graph-dee6f.18a8b494ec42bc45 namespace: kserve-ci-e2e-test resourceVersion: "19553" uid: 9c31b6e8-c6fb-4e58-b056-edab96db2f13 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:47:40.422972Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 1cb43a9a-3e61-4a9e-8a98-6db4b0437e65 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-055dc-predictor-54dcc78869-9lbzc to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc.18a8b52d97d2a1ba namespace: kserve-ci-e2e-test resourceVersion: "25096" uid: ee7708c5-37dd-4f10-b9ba-b925166c12af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc namespace: kserve-ci-e2e-test resourceVersion: "25103" uid: 1cb43a9a-3e61-4a9e-8a98-6db4b0437e65 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc.18a8b52d9ec2b917 namespace: kserve-ci-e2e-test resourceVersion: "25113" uid: 4ffab08c-2091-4ae8-b486-86736a4a2f89 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 1cb43a9a-3e61-4a9e-8a98-6db4b0437e65 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc.18a8b52da01593e6 namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: e59a6ea1-d26f-4df0-8103-1723d6a3b9c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 1cb43a9a-3e61-4a9e-8a98-6db4b0437e65 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc.18a8b52da36120a7 namespace: kserve-ci-e2e-test resourceVersion: "25117" uid: 41ec6b43-f09c-42ea-b136-5c77d825e8fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 1cb43a9a-3e61-4a9e-8a98-6db4b0437e65 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc.18a8b52da3d62ca3 namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: c852e1b5-93f1-4dc0-852e-9015857a4a93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:47:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 1cb43a9a-3e61-4a9e-8a98-6db4b0437e65 kind: Event lastTimestamp: "2026-04-22T14:48:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:22Z" name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc.18a8b52dcad3acf5 namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: f0603545-ca9d-4c9a-af5a-5f6c90c7f292 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 1cb43a9a-3e61-4a9e-8a98-6db4b0437e65 kind: Event lastTimestamp: "2026-04-22T14:57:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:57:05Z" name: error-404-isvc-055dc-predictor-54dcc78869-9lbzc.18a8b5b11057eb02 namespace: kserve-ci-e2e-test resourceVersion: "29568" uid: aef1c137-443b-4a6d-af06-672a01a1883a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-055dc-predictor-54dcc78869 namespace: kserve-ci-e2e-test resourceVersion: "25076" uid: bb30be86-2a97-4c0a-a49e-726adf6e46d7 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: 'Created pod: error-404-isvc-055dc-predictor-54dcc78869-9lbzc' metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-055dc-predictor-54dcc78869.18a8b52d976f10bc namespace: kserve-ci-e2e-test resourceVersion: "25093" uid: c46c5841-79d4-48df-91db-ac161a887722 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25073" uid: 7d3044d6-92aa-4622-9cda-8e924946288c kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Scaled up replica set error-404-isvc-055dc-predictor-54dcc78869 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-055dc-predictor.18a8b52d95d061cf namespace: kserve-ci-e2e-test resourceVersion: "25078" uid: 390bfbaf-4241-4ed2-9088-e7e4bd37a65c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:47:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: bcc4dd04-db1e-43af-ab11-e1e24843c1ff kind: Event lastTimestamp: "2026-04-22T14:48:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:47:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:25Z" name: error-404-isvc-055dc-predictor.18a8b531143d150b namespace: kserve-ci-e2e-test resourceVersion: "25564" uid: d996debc-4b80-4757-9960-04c9792148c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:47:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: bcc4dd04-db1e-43af-ab11-e1e24843c1ff kind: Event lastTimestamp: "2026-04-22T14:48:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:47:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:25Z" name: error-404-isvc-055dc-predictor.18a8b531149e4dbf namespace: kserve-ci-e2e-test resourceVersion: "25565" uid: a747eca5-d0f8-4eb6-9125-cd32a7eceb5a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:48:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: bcc4dd04-db1e-43af-ab11-e1e24843c1ff kind: Event lastTimestamp: "2026-04-22T14:48:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:55Z" name: error-404-isvc-055dc-predictor.18a8b53b91559645 namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: 4ced8eee-bc76-4c5f-b2f2-6c71c776691e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:48:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: bcc4dd04-db1e-43af-ab11-e1e24843c1ff kind: Event lastTimestamp: "2026-04-22T14:48:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:55Z" name: error-404-isvc-055dc-predictor.18a8b53b91b4501f namespace: kserve-ci-e2e-test resourceVersion: "25837" uid: 6b0492b3-9732-44a5-be9c-a710c9b1eb46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-055dc namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: d2d185ba-cba4-423c-a060-23c779537cd4 kind: Event lastTimestamp: "2026-04-22T14:48:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:48:32Z" name: error-404-isvc-055dc.18a8b52d95aadfae namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: be88f1b2-ec1d-48c1-b186-04620ac012df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-055dc namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: d2d185ba-cba4-423c-a060-23c779537cd4 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-055dc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-055dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-055dc.18a8b52d95f207d3 namespace: kserve-ci-e2e-test resourceVersion: "25080" uid: 5b7e2b93-fb06-4fb4-b17b-3b4f4eab0f1d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-055dc namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: d2d185ba-cba4-423c-a060-23c779537cd4 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-055dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-055dc.18a8b52d95f2f4f8 namespace: kserve-ci-e2e-test resourceVersion: "25082" uid: d50517c0-c4d5-4036-a824-bafdd5bce95b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-055dc namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: d2d185ba-cba4-423c-a060-23c779537cd4 kind: Event lastTimestamp: "2026-04-22T14:48:32Z" message: InferenceService [error-404-isvc-055dc] is Ready metadata: creationTimestamp: "2026-04-22T14:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:48:32Z" name: error-404-isvc-055dc.18a8b539af8c1a89 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: 881c5eb3-52b1-4d5e-8dfc-1ce87b24b77d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:37:34.739509Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: e386a23c-decf-4288-90e5-d43c0090ea28 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b.18a8b4a0924678bf namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: 4da614a3-7538-4b0c-8057-6752015fbf48 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: e386a23c-decf-4288-90e5-d43c0090ea28 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b.18a8b4a099394c43 namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: b76a75b8-8bb7-4711-af07-5e577bde349b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: e386a23c-decf-4288-90e5-d43c0090ea28 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b.18a8b4a09a56108b namespace: kserve-ci-e2e-test resourceVersion: "20263" uid: a9565da5-af49-4f87-b6ed-05a3ffb1713f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: e386a23c-decf-4288-90e5-d43c0090ea28 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b.18a8b4a09e54a5ac namespace: kserve-ci-e2e-test resourceVersion: "20265" uid: 2fed610c-2f04-4ecb-b8da-b0854362a13c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: e386a23c-decf-4288-90e5-d43c0090ea28 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b.18a8b4a09ec5dd50 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 4e7cabfd-f4e3-4999-8956-3363b2394881 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: e386a23c-decf-4288-90e5-d43c0090ea28 kind: Event lastTimestamp: "2026-04-22T14:38:16Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:16Z" name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b.18a8b4a0c3e54d36 namespace: kserve-ci-e2e-test resourceVersion: "20680" uid: 71cadc7f-2363-4a8b-b3b0-9da396bf4c22 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: e386a23c-decf-4288-90e5-d43c0090ea28 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:59Z" name: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b.18a8b5241d6c5ba1 namespace: kserve-ci-e2e-test resourceVersion: "24542" uid: ad7d047c-c97a-40cb-9e08-cf89a0715bf0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0eb25-predictor-f6f5cc7f5 namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: 7b503793-f4d3-4cc6-a3b7-499ef1407baf kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: 'Created pod: error-404-isvc-0eb25-predictor-f6f5cc7f5-fbr4b' metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-0eb25-predictor-f6f5cc7f5.18a8b4a091e468d1 namespace: kserve-ci-e2e-test resourceVersion: "20245" uid: ba4681bd-11ba-4a7a-9544-50b5ef27244a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0eb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: 5610fd5c-e856-49b8-b19e-75ef3736a61e kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Scaled up replica set error-404-isvc-0eb25-predictor-f6f5cc7f5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-0eb25-predictor.18a8b4a090972ef1 namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: 346d37a7-9faa-4214-90f5-d877d048eaf2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0eb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: 41f8b166-39c4-485f-9264-59f78773bbcd kind: Event lastTimestamp: "2026-04-22T14:38:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:49Z" name: error-404-isvc-0eb25-predictor.18a8b4a40f3642d3 namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: df2e8a19-75c3-4d25-99f2-24d670919bc4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0eb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: 41f8b166-39c4-485f-9264-59f78773bbcd kind: Event lastTimestamp: "2026-04-22T14:38:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:49Z" name: error-404-isvc-0eb25-predictor.18a8b4a40f9b2497 namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: d652f8bf-f810-4248-a9c7-23380f414747 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: d94bea8a-56da-4817-a9e2-2e10a98bd549 kind: Event lastTimestamp: "2026-04-22T14:38:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:38:26Z" name: error-404-isvc-0eb25.18a8b4a0909db176 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: e4b51c24-b824-4fdb-978c-cbc13dc37c1f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: d94bea8a-56da-4817-a9e2-2e10a98bd549 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: 'Failed to update status for InferenceService "error-404-isvc-0eb25": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0eb25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-0eb25.18a8b4a0910eb625 namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: 5fb5e9e4-04ee-42f4-bb73-a28b2c74844e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: d94bea8a-56da-4817-a9e2-2e10a98bd549 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0eb25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-0eb25.18a8b4a0910f01ef namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: 7c5fce92-00ae-44d2-adda-4d3c8832faa2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: d94bea8a-56da-4817-a9e2-2e10a98bd549 kind: Event lastTimestamp: "2026-04-22T14:38:26Z" message: InferenceService [error-404-isvc-0eb25] is Ready metadata: creationTimestamp: "2026-04-22T14:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:38:26Z" name: error-404-isvc-0eb25.18a8b4aca7855735 namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: ec022c4e-6da3-4d56-ba44-f6ff96a33f5b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:26:49.628510Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: a1340150-0c1a-4d0f-ae54-df2d114c41fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1091a-predictor-7f48745f5-pmhpc to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:26:49Z" name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc.18a8b40a5ea9ccb1 namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: ca413117-bb1d-4065-9215-546614d2d9ab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: a1340150-0c1a-4d0f-ae54-df2d114c41fa kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:26:49Z" name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc.18a8b40a65c9f2af namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 0ae9ce8d-f961-460d-a2a9-2cab86b03669 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a1340150-0c1a-4d0f-ae54-df2d114c41fa kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc.18a8b40a6740c225 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 10088840-9955-4d71-9735-797aeb84d1e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a1340150-0c1a-4d0f-ae54-df2d114c41fa kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc.18a8b40a6b47fb1d namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 75f06a99-88b4-4159-8548-2dd9d614595d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a1340150-0c1a-4d0f-ae54-df2d114c41fa kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc.18a8b40a6bcab086 namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: f541ab8f-8ede-482d-a62a-4fbc3aa26c95 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a1340150-0c1a-4d0f-ae54-df2d114c41fa kind: Event lastTimestamp: "2026-04-22T14:27:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:27:31Z" name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc.18a8b40a9fd228af namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: b4dc9834-0cc7-4b15-a1e0-99be7682e2b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: a1340150-0c1a-4d0f-ae54-df2d114c41fa kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-1091a-predictor-7f48745f5-pmhpc.18a8b48dda86fb51 namespace: kserve-ci-e2e-test resourceVersion: "19171" uid: 86e3e944-caac-4e1f-b666-378ce105c303 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1091a-predictor-7f48745f5 namespace: kserve-ci-e2e-test resourceVersion: "14877" uid: bb68c4ea-04b2-4b6c-8461-cefdb77d387c kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: 'Created pod: error-404-isvc-1091a-predictor-7f48745f5-pmhpc' metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:49Z" name: error-404-isvc-1091a-predictor-7f48745f5.18a8b40a5df3c3f7 namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 60d63f83-219c-4e25-b4e8-f9a9e094af5e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: 0de876e7-7f61-40c2-b19c-b7e115559854 kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Scaled up replica set error-404-isvc-1091a-predictor-7f48745f5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:49Z" name: error-404-isvc-1091a-predictor.18a8b40a5c70bcd1 namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: 80d99aca-460c-454e-b05b-67c560c2fe21 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:27:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 54df49a9-a783-438a-aa05-44a2f084fb10 kind: Event lastTimestamp: "2026-04-22T14:27:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:49Z" name: error-404-isvc-1091a-predictor.18a8b40ddb1e4abf namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: 49f8465d-786c-405d-bd30-bd480eac3e4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:27:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 54df49a9-a783-438a-aa05-44a2f084fb10 kind: Event lastTimestamp: "2026-04-22T14:27:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:49Z" name: error-404-isvc-1091a-predictor.18a8b40ddba0ffd4 namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 81388f09-4d2a-43d3-8eaa-d9a25aa2fc5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 54df49a9-a783-438a-aa05-44a2f084fb10 kind: Event lastTimestamp: "2026-04-22T14:28:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:04Z" name: error-404-isvc-1091a-predictor.18a8b41bd687a8c1 namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: 0e6f1d33-0c25-4343-aad8-efd6d39bc912 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 54df49a9-a783-438a-aa05-44a2f084fb10 kind: Event lastTimestamp: "2026-04-22T14:28:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:04Z" name: error-404-isvc-1091a-predictor.18a8b41bd6f0375c namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 4620e713-4217-4667-a91f-aa777ef7a277 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1091a namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: e6befaee-bca4-4fde-b62b-88634c967c8b kind: Event lastTimestamp: "2026-04-22T14:27:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:41Z" name: error-404-isvc-1091a.18a8b40a5c682983 namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 2930567b-4da0-45cc-9a53-343e878f2520 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1091a namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: e6befaee-bca4-4fde-b62b-88634c967c8b kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: 'Failed to update status for InferenceService "error-404-isvc-1091a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1091a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:49Z" name: error-404-isvc-1091a.18a8b40a5d0fd8a7 namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: 4d8fd79e-3ed1-43af-a0f9-2ae97433b03c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1091a namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: e6befaee-bca4-4fde-b62b-88634c967c8b kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1091a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:49Z" name: error-404-isvc-1091a.18a8b40a5d1026e2 namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: 6c8fe035-b8b6-4935-876a-a8d2170f1f90 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1091a namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: e6befaee-bca4-4fde-b62b-88634c967c8b kind: Event lastTimestamp: "2026-04-22T14:27:41Z" message: InferenceService [error-404-isvc-1091a] is Ready metadata: creationTimestamp: "2026-04-22T14:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:41Z" name: error-404-isvc-1091a.18a8b4168496e3bd namespace: kserve-ci-e2e-test resourceVersion: "15426" uid: 5071da05-cae3-4235-b446-4e8286252fbb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:24:39.474467Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh namespace: kserve-ci-e2e-test resourceVersion: "13330" uid: 6569b4b3-4678-49b0-86c5-21d7a839c64f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:24:39Z" name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh.18a8b3ec10e0abe1 namespace: kserve-ci-e2e-test resourceVersion: "13334" uid: 697158e1-c7bc-4ac0-b0f9-73e74e7e3626 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh namespace: kserve-ci-e2e-test resourceVersion: "13339" uid: 6569b4b3-4678-49b0-86c5-21d7a839c64f kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:24:39Z" name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh.18a8b3ec17b0f54b namespace: kserve-ci-e2e-test resourceVersion: "13361" uid: 49d510bb-53fe-4b4a-98f3-de120dc57b68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh namespace: kserve-ci-e2e-test resourceVersion: "13333" uid: 6569b4b3-4678-49b0-86c5-21d7a839c64f kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:39Z" name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh.18a8b3ec19950753 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: 39a862b4-345e-4c8d-aebc-482af606c446 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh namespace: kserve-ci-e2e-test resourceVersion: "13333" uid: 6569b4b3-4678-49b0-86c5-21d7a839c64f kind: Event lastTimestamp: "2026-04-22T14:24:53Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" in 13.668s (13.668s including waiting). Image size: 1334135903 bytes.' metadata: creationTimestamp: "2026-04-22T14:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:53Z" name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh.18a8b3ef484f840e namespace: kserve-ci-e2e-test resourceVersion: "13526" uid: 187a0966-20c9-4ecc-a79e-65626b306ba1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh namespace: kserve-ci-e2e-test resourceVersion: "13333" uid: 6569b4b3-4678-49b0-86c5-21d7a839c64f kind: Event lastTimestamp: "2026-04-22T14:24:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:53Z" name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh.18a8b3ef4c0aa0af namespace: kserve-ci-e2e-test resourceVersion: "13527" uid: c78cb681-e9c1-41d5-ad69-5d32d320e014 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh namespace: kserve-ci-e2e-test resourceVersion: "13333" uid: 6569b4b3-4678-49b0-86c5-21d7a839c64f kind: Event lastTimestamp: "2026-04-22T14:24:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:53Z" name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh.18a8b3ef4c774e28 namespace: kserve-ci-e2e-test resourceVersion: "13528" uid: 40991404-4e47-4e2f-9d76-8e35f2e4683d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh namespace: kserve-ci-e2e-test resourceVersion: "13333" uid: 6569b4b3-4678-49b0-86c5-21d7a839c64f kind: Event lastTimestamp: "2026-04-22T14:25:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:34Z" name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh.18a8b3ef4dc8b408 namespace: kserve-ci-e2e-test resourceVersion: "13926" uid: da728e1e-3140-42c1-b193-cac431460aa0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh namespace: kserve-ci-e2e-test resourceVersion: "13333" uid: 6569b4b3-4678-49b0-86c5-21d7a839c64f kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:09Z" name: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh.18a8b4010ddf4da0 namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 95c4b198-20c2-4f89-a711-ba620a4f161b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1cfdf-predictor-7c4489dd67 namespace: kserve-ci-e2e-test resourceVersion: "13322" uid: e6c00761-c27d-424d-9419-ecc753129f80 kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: 'Created pod: error-404-isvc-1cfdf-predictor-7c4489dd67-rxsqh' metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: error-404-isvc-1cfdf-predictor-7c4489dd67.18a8b3ec1067835b namespace: kserve-ci-e2e-test resourceVersion: "13331" uid: 26225bbb-6312-42bd-91af-a9b87ea10111 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13318" uid: 37dc0147-55e8-4a70-a80f-7d49c32c5ca3 kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Scaled up replica set error-404-isvc-1cfdf-predictor-7c4489dd67 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: error-404-isvc-1cfdf-predictor.18a8b3ec0efb0281 namespace: kserve-ci-e2e-test resourceVersion: "13325" uid: f023b74c-39c2-4bfc-ae2e-9893a4b33029 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: 86dce311-c683-4075-b414-038f91554ce0 kind: Event lastTimestamp: "2026-04-22T14:25:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:24Z" name: error-404-isvc-1cfdf-predictor.18a8b3ef8d7274b8 namespace: kserve-ci-e2e-test resourceVersion: "13848" uid: 16c2bdc9-19ab-42a4-b3f3-48af4f8c4250 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: 86dce311-c683-4075-b414-038f91554ce0 kind: Event lastTimestamp: "2026-04-22T14:25:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:24Z" name: error-404-isvc-1cfdf-predictor.18a8b3ef8def9c64 namespace: kserve-ci-e2e-test resourceVersion: "13849" uid: 21a2fbb2-172b-4620-a55b-76a03c4a513a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13575" uid: 86dce311-c683-4075-b414-038f91554ce0 kind: Event lastTimestamp: "2026-04-22T14:25:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:54Z" name: error-404-isvc-1cfdf-predictor.18a8b3fa0af5f53b namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 9db04c81-042e-4f26-9d8c-a9459c50e830 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13575" uid: 86dce311-c683-4075-b414-038f91554ce0 kind: Event lastTimestamp: "2026-04-22T14:25:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:54Z" name: error-404-isvc-1cfdf-predictor.18a8b3fa0bff77fb namespace: kserve-ci-e2e-test resourceVersion: "14110" uid: 16414270-53b0-4bbc-93d2-cc77161930ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1cfdf namespace: kserve-ci-e2e-test resourceVersion: "13304" uid: 6d9ef42f-5f21-4075-80a0-074c939f91b8 kind: Event lastTimestamp: "2026-04-22T14:25:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:25:44Z" name: error-404-isvc-1cfdf.18a8b3ec0ed62a4b namespace: kserve-ci-e2e-test resourceVersion: "14025" uid: 3fe5c66a-2bbc-4adc-982d-e11740193ab4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1cfdf namespace: kserve-ci-e2e-test resourceVersion: "14022" uid: 6d9ef42f-5f21-4075-80a0-074c939f91b8 kind: Event lastTimestamp: "2026-04-22T14:25:44Z" message: InferenceService [error-404-isvc-1cfdf] is Ready metadata: creationTimestamp: "2026-04-22T14:25:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:25:44Z" name: error-404-isvc-1cfdf.18a8b3fb32a2142f namespace: kserve-ci-e2e-test resourceVersion: "14023" uid: ff949f76-27f1-4c0b-a60d-29b4a67a0c22 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:36:14.530636Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th namespace: kserve-ci-e2e-test resourceVersion: "19225" uid: 0594a2e1-b605-4fc8-888a-9f53129071bf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th.18a8b48de5742852 namespace: kserve-ci-e2e-test resourceVersion: "19229" uid: d5d50ae2-8211-4b8e-9aa2-611b37f667f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: 0594a2e1-b605-4fc8-888a-9f53129071bf kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th.18a8b48dec5b65c0 namespace: kserve-ci-e2e-test resourceVersion: "19243" uid: 34c593d1-18a6-4e05-b72a-4e009c4694a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th namespace: kserve-ci-e2e-test resourceVersion: "19228" uid: 0594a2e1-b605-4fc8-888a-9f53129071bf kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th.18a8b48dedbb0a67 namespace: kserve-ci-e2e-test resourceVersion: "19250" uid: 140e88d0-e5db-429e-9f99-5d2dc3da00b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th namespace: kserve-ci-e2e-test resourceVersion: "19228" uid: 0594a2e1-b605-4fc8-888a-9f53129071bf kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th.18a8b48df21b44b9 namespace: kserve-ci-e2e-test resourceVersion: "19253" uid: 78fae288-f80c-4c7f-b384-1409ae7cdd18 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th namespace: kserve-ci-e2e-test resourceVersion: "19228" uid: 0594a2e1-b605-4fc8-888a-9f53129071bf kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th.18a8b48df28891bd namespace: kserve-ci-e2e-test resourceVersion: "19254" uid: 27d6361b-e8c6-4b82-8a76-e1525fe58653 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T14:36:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th namespace: kserve-ci-e2e-test resourceVersion: "19228" uid: 0594a2e1-b605-4fc8-888a-9f53129071bf kind: Event lastTimestamp: "2026-04-22T14:37:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:36:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:36Z" name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th.18a8b48e14a7d115 namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: 6bf20810-7d7f-4489-ba28-1b2340a7ccac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th namespace: kserve-ci-e2e-test resourceVersion: "19228" uid: 0594a2e1-b605-4fc8-888a-9f53129071bf kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:34Z" name: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th.18a8b4a0900e972d namespace: kserve-ci-e2e-test resourceVersion: "20235" uid: dd7ddaff-664b-429d-ab0c-b1acedd02d8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-33e8e-predictor-6697b7fc5f namespace: kserve-ci-e2e-test resourceVersion: "19218" uid: 16c57543-3cc9-4475-8d02-8cff670ea88c kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: 'Created pod: error-404-isvc-33e8e-predictor-6697b7fc5f-7j8th' metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-33e8e-predictor-6697b7fc5f.18a8b48de4bf1ead namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: b270217c-3aa7-4c74-9a54-71f961e78e2d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19216" uid: 96779608-2e29-4712-a7fd-d683a702da6e kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Scaled up replica set error-404-isvc-33e8e-predictor-6697b7fc5f from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-33e8e-predictor.18a8b48de3367f6b namespace: kserve-ci-e2e-test resourceVersion: "19220" uid: 59610f07-c299-4081-96d9-db139ae22bce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:36:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19217" uid: 3de3c0c2-a994-4b84-bfcb-2843986b0901 kind: Event lastTimestamp: "2026-04-22T14:36:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:44Z" name: error-404-isvc-33e8e-predictor.18a8b49161ab116a namespace: kserve-ci-e2e-test resourceVersion: "19525" uid: 2b267c3b-7df5-4811-976a-09cf5d7ce6e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:36:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19217" uid: 3de3c0c2-a994-4b84-bfcb-2843986b0901 kind: Event lastTimestamp: "2026-04-22T14:36:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:44Z" name: error-404-isvc-33e8e-predictor.18a8b491620db938 namespace: kserve-ci-e2e-test resourceVersion: "19526" uid: 8b95b660-c8ee-4b1d-be51-35e9a98a4691 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:36:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 3de3c0c2-a994-4b84-bfcb-2843986b0901 kind: Event lastTimestamp: "2026-04-22T14:37:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:29Z" name: error-404-isvc-33e8e-predictor.18a8b4985f82151c namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: f336b5f4-3e7c-40b4-8ae6-d6f73921cfda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:36:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 3de3c0c2-a994-4b84-bfcb-2843986b0901 kind: Event lastTimestamp: "2026-04-22T14:37:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:29Z" name: error-404-isvc-33e8e-predictor.18a8b4985fe13c95 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 9053d42a-885c-4800-8f23-b168d042beb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-33e8e namespace: kserve-ci-e2e-test resourceVersion: "19207" uid: 7f80a88f-23c6-4b35-98f8-0c2207906830 kind: Event lastTimestamp: "2026-04-22T14:37:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:06Z" name: error-404-isvc-33e8e.18a8b48de310ccce namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 87b1a65b-a725-438f-a6ce-eb6efdce0cbc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-33e8e namespace: kserve-ci-e2e-test resourceVersion: "19207" uid: 7f80a88f-23c6-4b35-98f8-0c2207906830 kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: 'Failed to update status for InferenceService "error-404-isvc-33e8e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-33e8e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-33e8e.18a8b48de397a957 namespace: kserve-ci-e2e-test resourceVersion: "19222" uid: 3a15f3b2-b6df-4c7c-9373-dd8b8299c2e6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-33e8e namespace: kserve-ci-e2e-test resourceVersion: "19207" uid: 7f80a88f-23c6-4b35-98f8-0c2207906830 kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-33e8e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:14Z" name: error-404-isvc-33e8e.18a8b48de3987b08 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 168b40fb-8628-47cc-8227-552636979f2f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-33e8e namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 7f80a88f-23c6-4b35-98f8-0c2207906830 kind: Event lastTimestamp: "2026-04-22T14:37:06Z" message: InferenceService [error-404-isvc-33e8e] is Ready metadata: creationTimestamp: "2026-04-22T14:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:06Z" name: error-404-isvc-33e8e.18a8b499f81f8e85 namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: 19e76ec9-9619-4f58-aec0-05fae304a78b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:36:54.922713Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: b29239c9-642b-4958-ae16-a2649264b49c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4547e-predictor-5b6cb6694f-b9snp to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:36:54Z" name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp.18a8b4974d025eba namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: d9d77928-708b-44a4-948c-8e0d9dd78ef1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: b29239c9-642b-4958-ae16-a2649264b49c kind: Event lastTimestamp: "2026-04-22T14:36:55Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:36:55Z" name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp.18a8b497540237f4 namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 301e1ebb-e170-4d34-893f-3391bd876f34 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: b29239c9-642b-4958-ae16-a2649264b49c kind: Event lastTimestamp: "2026-04-22T14:36:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:55Z" name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp.18a8b4975537a8f8 namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: f5b86f87-b96b-486c-a950-19c2daea89ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: b29239c9-642b-4958-ae16-a2649264b49c kind: Event lastTimestamp: "2026-04-22T14:36:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:55Z" name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp.18a8b49758882b6a namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: 26969cda-ec94-4fb2-b7d8-25961b498a63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: b29239c9-642b-4958-ae16-a2649264b49c kind: Event lastTimestamp: "2026-04-22T14:36:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:55Z" name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp.18a8b49758f49515 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: 4e379bb9-9eb8-48d9-bea0-2da5279bc3b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: b29239c9-642b-4958-ae16-a2649264b49c kind: Event lastTimestamp: "2026-04-22T14:37:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:36Z" name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp.18a8b4976c416e53 namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: 34f6188a-be2e-4e94-bfaa-b05251e26247 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: b29239c9-642b-4958-ae16-a2649264b49c kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:20Z" name: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp.18a8b51ae2d9e17b namespace: kserve-ci-e2e-test resourceVersion: "24164" uid: 17dcdcc2-e510-42e2-aa0f-f8a3e2127fec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4547e-predictor-5b6cb6694f namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: eb0c16ee-1368-4a0a-ab81-02c4695ac475 kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: 'Created pod: error-404-isvc-4547e-predictor-5b6cb6694f-b9snp' metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:54Z" name: error-404-isvc-4547e-predictor-5b6cb6694f.18a8b4974c8252ca namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 1666a41a-1392-4a73-b4d0-e56a9ac279eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: 9ecccd18-8684-48e0-a60c-f5abb3258179 kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: Scaled up replica set error-404-isvc-4547e-predictor-5b6cb6694f from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:54Z" name: error-404-isvc-4547e-predictor.18a8b4974b21633c namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 929f9aa2-4c01-4daf-aa34-e337578d2381 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: 93440c2a-0e77-45cf-95b8-63f033b04bc0 kind: Event lastTimestamp: "2026-04-22T14:37:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:09Z" name: error-404-isvc-4547e-predictor.18a8b49ac9c3e4de namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: 17d2774e-a99c-43e9-b3a8-b30fbeed2ca7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: 93440c2a-0e77-45cf-95b8-63f033b04bc0 kind: Event lastTimestamp: "2026-04-22T14:37:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:09Z" name: error-404-isvc-4547e-predictor.18a8b49aca2ae4e8 namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 8d7fc96b-7ef6-4026-b23e-f0857b8ee818 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:37:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19926" uid: 93440c2a-0e77-45cf-95b8-63f033b04bc0 kind: Event lastTimestamp: "2026-04-22T14:38:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:09Z" name: error-404-isvc-4547e-predictor.18a8b49e48cfd5ee namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 9027e45b-7fd4-4196-9627-80a8a632e555 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:37:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19926" uid: 93440c2a-0e77-45cf-95b8-63f033b04bc0 kind: Event lastTimestamp: "2026-04-22T14:38:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:09Z" name: error-404-isvc-4547e-predictor.18a8b49e492f7184 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: e946ae38-7100-4ba0-ae2a-1b72f429d717 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4547e namespace: kserve-ci-e2e-test resourceVersion: "19694" uid: b16e0008-f456-4c87-adfb-fb98f6b2d723 kind: Event lastTimestamp: "2026-04-22T14:37:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:46Z" name: error-404-isvc-4547e.18a8b4974b20f7e4 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: b9eaf66b-056b-48df-b70c-e4dd739afe76 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4547e namespace: kserve-ci-e2e-test resourceVersion: "19694" uid: b16e0008-f456-4c87-adfb-fb98f6b2d723 kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: 'Failed to update status for InferenceService "error-404-isvc-4547e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4547e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:54Z" name: error-404-isvc-4547e.18a8b4974b95ed88 namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: 87324633-191b-49ca-96b6-3f3582dea75a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4547e namespace: kserve-ci-e2e-test resourceVersion: "19694" uid: b16e0008-f456-4c87-adfb-fb98f6b2d723 kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4547e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:54Z" name: error-404-isvc-4547e.18a8b4974b965e7e namespace: kserve-ci-e2e-test resourceVersion: "19730" uid: 8599f986-7dcd-462b-aae3-953db342141c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4547e namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: b16e0008-f456-4c87-adfb-fb98f6b2d723 kind: Event lastTimestamp: "2026-04-22T14:37:46Z" message: InferenceService [error-404-isvc-4547e] is Ready metadata: creationTimestamp: "2026-04-22T14:37:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:46Z" name: error-404-isvc-4547e.18a8b4a34ffdd935 namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 006616af-b023-47fa-bf2b-57d8a7e45add reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:46:59.834220Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9 namespace: kserve-ci-e2e-test resourceVersion: "24581" uid: 8fc9f5d6-3ebf-4867-a668-9d844efd1d0f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4e291-predictor-84859f6fd9-kzwr9 to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:46:59Z" name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9.18a8b524248b709d namespace: kserve-ci-e2e-test resourceVersion: "24585" uid: 5aa8a70f-41ba-447f-b935-af80bf557f17 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9 namespace: kserve-ci-e2e-test resourceVersion: "24591" uid: 8fc9f5d6-3ebf-4867-a668-9d844efd1d0f kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:46:59Z" name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9.18a8b5242bb351fc namespace: kserve-ci-e2e-test resourceVersion: "24598" uid: d2ff242e-c547-4439-bd17-3dbfd50e0ef2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9 namespace: kserve-ci-e2e-test resourceVersion: "24583" uid: 8fc9f5d6-3ebf-4867-a668-9d844efd1d0f kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:59Z" name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9.18a8b5242cfa521e namespace: kserve-ci-e2e-test resourceVersion: "24602" uid: 3a2d0e31-9ea2-44df-9a5b-cbd2bb49f487 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9 namespace: kserve-ci-e2e-test resourceVersion: "24583" uid: 8fc9f5d6-3ebf-4867-a668-9d844efd1d0f kind: Event lastTimestamp: "2026-04-22T14:47:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:00Z" name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9.18a8b52430df0291 namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 878c1b7e-ed10-45b1-86c3-d2382b6fa088 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9 namespace: kserve-ci-e2e-test resourceVersion: "24583" uid: 8fc9f5d6-3ebf-4867-a668-9d844efd1d0f kind: Event lastTimestamp: "2026-04-22T14:47:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:00Z" name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9.18a8b524314dc6af namespace: kserve-ci-e2e-test resourceVersion: "24606" uid: 5355f2e2-0089-4f7c-91ac-0443fdfa97b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T14:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9 namespace: kserve-ci-e2e-test resourceVersion: "24583" uid: 8fc9f5d6-3ebf-4867-a668-9d844efd1d0f kind: Event lastTimestamp: "2026-04-22T15:04:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T15:04:31Z" name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9.18a8b52437b41ff2 namespace: kserve-ci-e2e-test resourceVersion: "32693" uid: 91221939-486c-4b53-a748-6a21b3f98aa9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T15:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9 namespace: kserve-ci-e2e-test resourceVersion: "24583" uid: 8fc9f5d6-3ebf-4867-a668-9d844efd1d0f kind: Event lastTimestamp: "2026-04-22T15:04:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T15:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T15:04:29Z" name: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9.18a8b61883ee4099 namespace: kserve-ci-e2e-test resourceVersion: "32647" uid: 8460b5bd-1f60-4912-a7ef-bf5f5391583b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4e291-predictor-84859f6fd9 namespace: kserve-ci-e2e-test resourceVersion: "24573" uid: 6f26e2f0-657f-4497-9c73-d76d00ce050c kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: 'Created pod: error-404-isvc-4e291-predictor-84859f6fd9-kzwr9' metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:59Z" name: error-404-isvc-4e291-predictor-84859f6fd9.18a8b52424178629 namespace: kserve-ci-e2e-test resourceVersion: "24582" uid: 952a8653-6f52-4032-bdf3-64e29a15230d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24570" uid: 72a9dfef-9ee2-4a31-84c8-60e8369cff7b kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: Scaled up replica set error-404-isvc-4e291-predictor-84859f6fd9 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:59Z" name: error-404-isvc-4e291-predictor.18a8b52422ab5e07 namespace: kserve-ci-e2e-test resourceVersion: "24574" uid: 1f8e45c1-6881-4ecb-87fc-9826b8f75a6a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:47:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24572" uid: b928fb90-c13e-47b9-bdf9-0d1a4282b918 kind: Event lastTimestamp: "2026-04-22T14:47:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:47:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:44Z" name: error-404-isvc-4e291-predictor.18a8b527a14e47ba namespace: kserve-ci-e2e-test resourceVersion: "25177" uid: adf661d6-a516-4591-8601-3ead97499721 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:47:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24572" uid: b928fb90-c13e-47b9-bdf9-0d1a4282b918 kind: Event lastTimestamp: "2026-04-22T14:47:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:47:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:44Z" name: error-404-isvc-4e291-predictor.18a8b527a1ba20a7 namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: 1b194e7f-aac7-4a10-9198-076adce659d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:47:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24769" uid: b928fb90-c13e-47b9-bdf9-0d1a4282b918 kind: Event lastTimestamp: "2026-04-22T14:48:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:14Z" name: error-404-isvc-4e291-predictor.18a8b5321de869dd namespace: kserve-ci-e2e-test resourceVersion: "25475" uid: f8cfbf03-aeea-4bd8-8f9e-d6deabe1f6c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:47:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24769" uid: b928fb90-c13e-47b9-bdf9-0d1a4282b918 kind: Event lastTimestamp: "2026-04-22T14:48:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:14Z" name: error-404-isvc-4e291-predictor.18a8b5321e4e1ca7 namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: 79e69dcf-a095-480e-9ca9-820be5d1263d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4e291 namespace: kserve-ci-e2e-test resourceVersion: "24544" uid: 820c232a-b543-4704-a799-4c0245a393e5 kind: Event lastTimestamp: "2026-04-22T14:47:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:51Z" name: error-404-isvc-4e291.18a8b52422ae5b53 namespace: kserve-ci-e2e-test resourceVersion: "25237" uid: 3513bfaa-5f78-40b9-8922-cac4b55337cc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4e291 namespace: kserve-ci-e2e-test resourceVersion: "24544" uid: 820c232a-b543-4704-a799-4c0245a393e5 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: 'Failed to update status for InferenceService "error-404-isvc-4e291": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4e291": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:46:59Z" name: error-404-isvc-4e291.18a8b524234d4f24 namespace: kserve-ci-e2e-test resourceVersion: "24578" uid: 5eef1723-0bfb-4516-b54c-2631f06a0c66 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4e291 namespace: kserve-ci-e2e-test resourceVersion: "24544" uid: 820c232a-b543-4704-a799-4c0245a393e5 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4e291": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:46:59Z" name: error-404-isvc-4e291.18a8b524234d9bcc namespace: kserve-ci-e2e-test resourceVersion: "24580" uid: 3ff80821-55ea-4d5a-8235-bc81e2f97580 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4e291 namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 820c232a-b543-4704-a799-4c0245a393e5 kind: Event lastTimestamp: "2026-04-22T14:47:51Z" message: InferenceService [error-404-isvc-4e291] is Ready metadata: creationTimestamp: "2026-04-22T14:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:51Z" name: error-404-isvc-4e291.18a8b5301b892939 namespace: kserve-ci-e2e-test resourceVersion: "25235" uid: 88dbd153-6c6e-4e7e-8ac7-db0cfa000d48 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:26:09.750735Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 9c12b754-7e62-456f-bb3d-f1d1940235c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:26:09Z" name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t.18a8b40115c33f34 namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: 164efdc4-0abc-4bd6-bcf3-34057c555ea9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: 9c12b754-7e62-456f-bb3d-f1d1940235c9 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:26:09Z" name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t.18a8b4011cd5a48d namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: aa6e1f8e-7b19-48b3-8830-84c40217d0b3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 9c12b754-7e62-456f-bb3d-f1d1940235c9 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:09Z" name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t.18a8b4011eb99413 namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: 736db420-8175-40c6-9b10-64367e3955d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 9c12b754-7e62-456f-bb3d-f1d1940235c9 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:09Z" name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t.18a8b40121d64f1f namespace: kserve-ci-e2e-test resourceVersion: "14387" uid: fe23346f-c8c4-4502-9af7-5de707fbe64a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 9c12b754-7e62-456f-bb3d-f1d1940235c9 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:09Z" name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t.18a8b40122426b9b namespace: kserve-ci-e2e-test resourceVersion: "14388" uid: 767ccb68-2d25-4db6-8d95-303d7d5d1b29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 9c12b754-7e62-456f-bb3d-f1d1940235c9 kind: Event lastTimestamp: "2026-04-22T14:26:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:51Z" name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t.18a8b40148d7dc20 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 72066d0d-21ce-46e9-a253-f307a0706111 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 9c12b754-7e62-456f-bb3d-f1d1940235c9 kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:34Z" name: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t.18a8b48499a85da5 namespace: kserve-ci-e2e-test resourceVersion: "18773" uid: 44775afd-343c-40b1-b56e-c349f3d6698e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-58a83-predictor-6b4556c5b5 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 0d0fec57-4a04-4e94-bb34-fae9276e809c kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: 'Created pod: error-404-isvc-58a83-predictor-6b4556c5b5-x4q2t' metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:09Z" name: error-404-isvc-58a83-predictor-6b4556c5b5.18a8b401152d14bf namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: 55c44365-7255-468d-976a-af935abf531f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: a4082d7d-043e-4b0d-b7cb-0ddf314c60bf kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Scaled up replica set error-404-isvc-58a83-predictor-6b4556c5b5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:09Z" name: error-404-isvc-58a83-predictor.18a8b40113536f34 namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: 8ee4d223-c036-466e-8326-0815f7a24cb6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 5947fde1-0048-49b3-9091-ad8f5d6ed5ac kind: Event lastTimestamp: "2026-04-22T14:27:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:24Z" name: error-404-isvc-58a83-predictor.18a8b40491cc15b1 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 44335951-4b6f-4e7f-875c-e971166a24ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 5947fde1-0048-49b3-9091-ad8f5d6ed5ac kind: Event lastTimestamp: "2026-04-22T14:27:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:24Z" name: error-404-isvc-58a83-predictor.18a8b404923c0932 namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: db51a6db-eba4-4707-a708-42e297acd02c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:26:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: 5947fde1-0048-49b3-9091-ad8f5d6ed5ac kind: Event lastTimestamp: "2026-04-22T14:27:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:09Z" name: error-404-isvc-58a83-predictor.18a8b40810ffb263 namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 616d9c19-7100-4efd-b0f8-97f10886553a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:26:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: 5947fde1-0048-49b3-9091-ad8f5d6ed5ac kind: Event lastTimestamp: "2026-04-22T14:27:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:09Z" name: error-404-isvc-58a83-predictor.18a8b408116e44ba namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: 8cd15288-1d9b-4550-a282-1478eea91c10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-58a83 namespace: kserve-ci-e2e-test resourceVersion: "14339" uid: 33fb5bdb-3760-41c5-adae-0092ec67d4cb kind: Event lastTimestamp: "2026-04-22T14:27:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:01Z" name: error-404-isvc-58a83.18a8b401132a68d5 namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 1472cc31-f711-4fcf-a395-2046c4a07aa1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-58a83 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: 33fb5bdb-3760-41c5-adae-0092ec67d4cb kind: Event lastTimestamp: "2026-04-22T14:27:01Z" message: InferenceService [error-404-isvc-58a83] is Ready metadata: creationTimestamp: "2026-04-22T14:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:01Z" name: error-404-isvc-58a83.18a8b40d2d2f0ca1 namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: 0d65df06-bd1f-48c4-881f-4998ca565d89 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:35:34.723316Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x namespace: kserve-ci-e2e-test resourceVersion: "18797" uid: 158c988e-59d6-4cda-8621-d2656f8e5850 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dee6f-predictor-59f85ff5f-4799x to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:35:34Z" name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x.18a8b484a0c09b04 namespace: kserve-ci-e2e-test resourceVersion: "18801" uid: d487f8fd-cedd-41e3-95e5-f4d7cde8882f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x namespace: kserve-ci-e2e-test resourceVersion: "18807" uid: 158c988e-59d6-4cda-8621-d2656f8e5850 kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:35:34Z" name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x.18a8b484a7b0c2cb namespace: kserve-ci-e2e-test resourceVersion: "18817" uid: f8ceefae-6652-405d-b504-e7490f06a548 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x namespace: kserve-ci-e2e-test resourceVersion: "18799" uid: 158c988e-59d6-4cda-8621-d2656f8e5850 kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:34Z" name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x.18a8b484a8f26ae2 namespace: kserve-ci-e2e-test resourceVersion: "18819" uid: 1ae7a2c1-4523-4932-b2df-8457509b10d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x namespace: kserve-ci-e2e-test resourceVersion: "18799" uid: 158c988e-59d6-4cda-8621-d2656f8e5850 kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:34Z" name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x.18a8b484adb85bb5 namespace: kserve-ci-e2e-test resourceVersion: "18823" uid: 5939894b-303b-43b8-a59b-fcd6be1bd480 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x namespace: kserve-ci-e2e-test resourceVersion: "18799" uid: 158c988e-59d6-4cda-8621-d2656f8e5850 kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:34Z" name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x.18a8b484ae27351d namespace: kserve-ci-e2e-test resourceVersion: "18825" uid: 979f0fd4-e944-4e53-b0c2-55373eb07ec8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T14:35:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x namespace: kserve-ci-e2e-test resourceVersion: "18799" uid: 158c988e-59d6-4cda-8621-d2656f8e5850 kind: Event lastTimestamp: "2026-04-22T14:36:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:56Z" name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x.18a8b484bd2ff8e2 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 6eede121-3ffb-485c-b610-3d01a991a025 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x namespace: kserve-ci-e2e-test resourceVersion: "18799" uid: 158c988e-59d6-4cda-8621-d2656f8e5850 kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:54Z" name: error-404-isvc-dee6f-predictor-59f85ff5f-4799x.18a8b4974bdbdd0e namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 7e6b8232-1d34-4668-b6ea-99a49040074e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dee6f-predictor-59f85ff5f namespace: kserve-ci-e2e-test resourceVersion: "18788" uid: bf3ffa95-bf38-48d7-84b2-1b91ab9a7a70 kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: 'Created pod: error-404-isvc-dee6f-predictor-59f85ff5f-4799x' metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:34Z" name: error-404-isvc-dee6f-predictor-59f85ff5f.18a8b484a023ad3d namespace: kserve-ci-e2e-test resourceVersion: "18798" uid: 5d101534-db59-4a64-9193-09c9d7a51bbc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18785" uid: 3673f9fd-2c58-4783-9ddd-48adf02ecf6a kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Scaled up replica set error-404-isvc-dee6f-predictor-59f85ff5f from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:34Z" name: error-404-isvc-dee6f-predictor.18a8b4849df7e14c namespace: kserve-ci-e2e-test resourceVersion: "18791" uid: c49ec3eb-d31f-4f93-8f76-93e3aa89f343 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18787" uid: 521fe7dd-7e4e-4a18-b305-81fa23dc3588 kind: Event lastTimestamp: "2026-04-22T14:36:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:35:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:04Z" name: error-404-isvc-dee6f-predictor.18a8b4881ca35d7a namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 49ae7686-2a98-4b47-bd5f-c4880d9a23d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18787" uid: 521fe7dd-7e4e-4a18-b305-81fa23dc3588 kind: Event lastTimestamp: "2026-04-22T14:36:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:35:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:04Z" name: error-404-isvc-dee6f-predictor.18a8b4881d0526a6 namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: c5502a05-1df3-4536-9aa4-9b80f366809e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 521fe7dd-7e4e-4a18-b305-81fa23dc3588 kind: Event lastTimestamp: "2026-04-22T14:36:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:49Z" name: error-404-isvc-dee6f-predictor.18a8b48f1a941375 namespace: kserve-ci-e2e-test resourceVersion: "19603" uid: 61f2523e-efe2-4f8c-9d32-7bbba6e6f45c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 521fe7dd-7e4e-4a18-b305-81fa23dc3588 kind: Event lastTimestamp: "2026-04-22T14:36:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:49Z" name: error-404-isvc-dee6f-predictor.18a8b48f1af3d149 namespace: kserve-ci-e2e-test resourceVersion: "19604" uid: 9261927e-e897-4ed5-8d22-d3b0673b3e90 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dee6f namespace: kserve-ci-e2e-test resourceVersion: "18771" uid: 429580f0-e0bc-4a34-b396-b013e41e9a8c kind: Event lastTimestamp: "2026-04-22T14:36:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:26Z" name: error-404-isvc-dee6f.18a8b4849df83951 namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: dad37b49-b440-40dc-83a0-b30bdd40033d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dee6f namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: 429580f0-e0bc-4a34-b396-b013e41e9a8c kind: Event lastTimestamp: "2026-04-22T14:36:26Z" message: InferenceService [error-404-isvc-dee6f] is Ready metadata: creationTimestamp: "2026-04-22T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:26Z" name: error-404-isvc-dee6f.18a8b490a0a9f28e namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: d188b06b-10ce-4f61-9289-17c6045c3682 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:46:20.067199Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq namespace: kserve-ci-e2e-test resourceVersion: "24149" uid: 76d4c1fb-542a-4c62-9d43-ba68f3c25ce8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f4200-predictor-86dfdc759f-mxddq to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:46:20Z" name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq.18a8b51ae23edade namespace: kserve-ci-e2e-test resourceVersion: "24158" uid: 03aaae45-cc19-4b7e-b622-c42ddab32b75 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq namespace: kserve-ci-e2e-test resourceVersion: "24165" uid: 76d4c1fb-542a-4c62-9d43-ba68f3c25ce8 kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:46:20Z" name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq.18a8b51ae90e1ddb namespace: kserve-ci-e2e-test resourceVersion: "24171" uid: 1f67ca3b-2f9d-4001-88ed-0053c52f3a94 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: 76d4c1fb-542a-4c62-9d43-ba68f3c25ce8 kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:20Z" name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq.18a8b51aea5c0357 namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: 8c36e2b2-c602-4b69-b044-be2a79633480 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: 76d4c1fb-542a-4c62-9d43-ba68f3c25ce8 kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:20Z" name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq.18a8b51aed0a6a1e namespace: kserve-ci-e2e-test resourceVersion: "24178" uid: 597040a9-38ec-41f8-852b-8866d366a49d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: 76d4c1fb-542a-4c62-9d43-ba68f3c25ce8 kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:20Z" name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq.18a8b51aed7363d0 namespace: kserve-ci-e2e-test resourceVersion: "24179" uid: 6f053650-e915-4ba1-9638-526926455fdf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: 76d4c1fb-542a-4c62-9d43-ba68f3c25ce8 kind: Event lastTimestamp: "2026-04-22T14:47:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:42Z" name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq.18a8b51b1c1351da namespace: kserve-ci-e2e-test resourceVersion: "25150" uid: ac15f159-c1a9-48f9-bc1f-4bd72884608a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: 76d4c1fb-542a-4c62-9d43-ba68f3c25ce8 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:40Z" name: error-404-isvc-f4200-predictor-86dfdc759f-mxddq.18a8b52d9934b323 namespace: kserve-ci-e2e-test resourceVersion: "25104" uid: 9911b5e3-2979-4873-8dd6-ed6d5bae6c21 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f4200-predictor-86dfdc759f namespace: kserve-ci-e2e-test resourceVersion: "24127" uid: 88ac5527-1629-4677-b58b-35755613efb0 kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: 'Created pod: error-404-isvc-f4200-predictor-86dfdc759f-mxddq' metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:20Z" name: error-404-isvc-f4200-predictor-86dfdc759f.18a8b51ae1857b7e namespace: kserve-ci-e2e-test resourceVersion: "24151" uid: 2e6e54db-c799-4ec3-bd8b-b3e89e08738b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 6a78193d-492a-4312-9b2b-7f2db4004597 kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: Scaled up replica set error-404-isvc-f4200-predictor-86dfdc759f from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:20Z" name: error-404-isvc-f4200-predictor.18a8b51ade4de293 namespace: kserve-ci-e2e-test resourceVersion: "24131" uid: 4d39de1b-743f-4e86-85cb-04f005132e91 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:46:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24126" uid: 81c659c3-1342-4fe0-8f69-a0fb1bf52a51 kind: Event lastTimestamp: "2026-04-22T14:46:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:50Z" name: error-404-isvc-f4200-predictor.18a8b51e5ce56a46 namespace: kserve-ci-e2e-test resourceVersion: "24420" uid: 51279f81-4bc3-4cc5-99c4-0437b842654e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:46:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24126" uid: 81c659c3-1342-4fe0-8f69-a0fb1bf52a51 kind: Event lastTimestamp: "2026-04-22T14:46:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:50Z" name: error-404-isvc-f4200-predictor.18a8b51e5d4414b3 namespace: kserve-ci-e2e-test resourceVersion: "24421" uid: 6e3e9194-89ee-439e-b504-a991fa8f7584 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:47:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24312" uid: 81c659c3-1342-4fe0-8f69-a0fb1bf52a51 kind: Event lastTimestamp: "2026-04-22T14:47:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:20Z" name: error-404-isvc-f4200-predictor.18a8b5255aab4084 namespace: kserve-ci-e2e-test resourceVersion: "24808" uid: f39e18f2-f1cd-45bc-b23a-a610d123079f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:47:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24312" uid: 81c659c3-1342-4fe0-8f69-a0fb1bf52a51 kind: Event lastTimestamp: "2026-04-22T14:47:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:20Z" name: error-404-isvc-f4200-predictor.18a8b5255b29cda8 namespace: kserve-ci-e2e-test resourceVersion: "24809" uid: c3137291-ec70-4f75-99d6-0d93b112f047 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:46:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f4200 namespace: kserve-ci-e2e-test resourceVersion: "24116" uid: 1e30c05c-de02-4580-9978-5d5e5d778850 kind: Event lastTimestamp: "2026-04-22T14:47:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:02Z" name: error-404-isvc-f4200.18a8b51ade1c4e7d namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: c1aaf158-0d1d-4052-91f6-fbcd3477821b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f4200 namespace: kserve-ci-e2e-test resourceVersion: "24650" uid: 1e30c05c-de02-4580-9978-5d5e5d778850 kind: Event lastTimestamp: "2026-04-22T14:47:02Z" message: InferenceService [error-404-isvc-f4200] is Ready metadata: creationTimestamp: "2026-04-22T14:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:02Z" name: error-404-isvc-f4200.18a8b524ac0dd7b0 namespace: kserve-ci-e2e-test resourceVersion: "24651" uid: 954f2578-b16f-4eda-b577-1d4bbda72af2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:24:39.406587Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13306" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:24:39Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3ec0cd4f506 namespace: kserve-ci-e2e-test resourceVersion: "13311" uid: 4d071d34-41cb-43ef-9311-19bed026b28b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13321" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:24:39Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3ec2a1598f5 namespace: kserve-ci-e2e-test resourceVersion: "13398" uid: dc0c09dd-f52e-4d48-a200-3b6b5ec4b0f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:39Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3ec2be04bd9 namespace: kserve-ci-e2e-test resourceVersion: "13402" uid: 96d4ca20-5fab-4630-b0e3-ac15c189a764 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:24:52Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" in 12.495s (12.495s including waiting). Image size: 299663288 bytes.' metadata: creationTimestamp: "2026-04-22T14:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:52Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3ef14a44c93 namespace: kserve-ci-e2e-test resourceVersion: "13517" uid: 56ad5293-acc7-464b-9381-ace94b45da3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:24:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:54Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3ef7477893e namespace: kserve-ci-e2e-test resourceVersion: "13544" uid: 37a3ee3a-b724-4d41-a09b-ffbbd5b8d3f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:24:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:54Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3ef74eda1dd namespace: kserve-ci-e2e-test resourceVersion: "13545" uid: 2b29fc6a-532a-4e87-b4df-240ca61886b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:24:58Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" metadata: creationTimestamp: "2026-04-22T14:24:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:58Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3f076b488b1 namespace: kserve-ci-e2e-test resourceVersion: "13617" uid: c15742e5-3a30-4c5a-987b-df672d0273f4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:25:06Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" in 8.491s (8.491s including waiting). Image size: 1562514855 bytes.' metadata: creationTimestamp: "2026-04-22T14:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:06Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3f270d0af2c namespace: kserve-ci-e2e-test resourceVersion: "13690" uid: 9a31aa69-3c93-4b8a-a1c7-4773f66914ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:25:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:06Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3f274951721 namespace: kserve-ci-e2e-test resourceVersion: "13692" uid: 256fb008-cf1f-461c-b560-4a995c1b92f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:25:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:06Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3f275088392 namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: 26b2d24a-3acf-431e-9fa5-b142e03860e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T14:25:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:26:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:25:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:08Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b3f29152bc74 namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: b425b91d-216f-45b3-8bdb-0f3e640088e3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8 namespace: kserve-ci-e2e-test resourceVersion: "13309" uid: 12046a98-409e-4b3f-acb6-380b2dfdac5a kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8.18a8b40a5b585098 namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: 83eddd85-268f-4b31-9047-8b308a1e251f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6d86c7594 namespace: kserve-ci-e2e-test resourceVersion: "13298" uid: 4137d8b6-091b-4815-9eab-8d60c9f62fe7 kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6d86c7594-9vjr8' metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: isvc-sklearn-graph-1-predictor-6d86c7594.18a8b3ec0c340b3e namespace: kserve-ci-e2e-test resourceVersion: "13307" uid: 478d83c2-2549-406e-aebc-417160b6cffd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13295" uid: b9e0edcc-34f3-4a18-bda3-bbbaf4eb5c48 kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6d86c7594 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: isvc-sklearn-graph-1-predictor.18a8b3ec09c46666 namespace: kserve-ci-e2e-test resourceVersion: "13301" uid: 2596fbdb-0561-4238-8d32-d33142a63702 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: 64801e22-a8ac-4a91-a649-aa2785d43eb2 kind: Event lastTimestamp: "2026-04-22T14:25:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:24Z" name: isvc-sklearn-graph-1-predictor.18a8b3ef88583c6c namespace: kserve-ci-e2e-test resourceVersion: "13845" uid: 3863950c-2032-4aa6-938a-0b1d2477ac71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: 64801e22-a8ac-4a91-a649-aa2785d43eb2 kind: Event lastTimestamp: "2026-04-22T14:25:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:24Z" name: isvc-sklearn-graph-1-predictor.18a8b3ef8916afc5 namespace: kserve-ci-e2e-test resourceVersion: "13846" uid: c0c88603-a692-4824-8973-b1480b235b36 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13560" uid: 64801e22-a8ac-4a91-a649-aa2785d43eb2 kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:39Z" name: isvc-sklearn-graph-1-predictor.18a8b3fa069b1c97 namespace: kserve-ci-e2e-test resourceVersion: "14694" uid: 5615f8f5-7901-498c-8f4c-3f704a1f71de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13560" uid: 64801e22-a8ac-4a91-a649-aa2785d43eb2 kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:39Z" name: isvc-sklearn-graph-1-predictor.18a8b3fa07933100 namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: 755c7c34-92e0-4484-9b96-050dc2539e28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: 42286699-ed7c-443b-a203-37107dde6b3d kind: Event lastTimestamp: "2026-04-22T14:26:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:18Z" name: isvc-sklearn-graph-1.18a8b3ec0977d1d1 namespace: kserve-ci-e2e-test resourceVersion: "14491" uid: f4e90d8e-a429-490b-8a5e-bc6f30eb22cd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 42286699-ed7c-443b-a203-37107dde6b3d kind: Event lastTimestamp: "2026-04-22T14:26:18Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-22T14:26:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:18Z" name: isvc-sklearn-graph-1.18a8b40320d39191 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 26ef1092-f0eb-4cf0-816a-2ddb4bf61bd3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:24:39.703115Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13386" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:24:39Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3ec1e8194a0 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 7d813ff9-75cb-4f12-b65e-7a94dcafebb9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13394" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:24:40Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:24:40Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3ec3cb303a1 namespace: kserve-ci-e2e-test resourceVersion: "13412" uid: fc8b00aa-4bc8-4a7b-bf46-f69bc6990631 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:24:40Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" metadata: creationTimestamp: "2026-04-22T14:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:40Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3ec3f9c8daa namespace: kserve-ci-e2e-test resourceVersion: "13415" uid: f8e0fdd1-fd08-4e6a-84ca-396f940022f2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:24:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" in 13.709s (13.709s including waiting). Image size: 299663288 bytes.' metadata: creationTimestamp: "2026-04-22T14:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:53Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3ef70c5e750 namespace: kserve-ci-e2e-test resourceVersion: "13542" uid: 8ebc140e-7eb9-41fb-9e62-7169a45ce3cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:24:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:54Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3ef746b6685 namespace: kserve-ci-e2e-test resourceVersion: "13543" uid: bbd4eeac-cd4c-45f4-a2ba-26cffc90582a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:24:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:54Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3ef74faabfe namespace: kserve-ci-e2e-test resourceVersion: "13546" uid: 3fa74612-aa16-4f74-ae13-670400cb371f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:24:58Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" metadata: creationTimestamp: "2026-04-22T14:24:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:58Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3f076a1fd46 namespace: kserve-ci-e2e-test resourceVersion: "13615" uid: 06a47957-234e-4426-9dc8-e981aa74c436 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:25:06Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" in 8.476s (8.476s including waiting). Image size: 1562514855 bytes.' metadata: creationTimestamp: "2026-04-22T14:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:06Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3f26fe38528 namespace: kserve-ci-e2e-test resourceVersion: "13689" uid: 9cffbbc5-0dcc-46a2-83d8-108853b6f67a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:25:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:06Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3f2745298c7 namespace: kserve-ci-e2e-test resourceVersion: "13691" uid: c0871510-0e6b-4a6c-9f78-50201707ebe9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:25:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:06Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3f274c9cfa9 namespace: kserve-ci-e2e-test resourceVersion: "13693" uid: e475d565-12a8-4e4e-a7f3-6acbdbec3178 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T14:25:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:26:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:25:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:08Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b3f29145ca3b namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 7f3d52c1-417d-4bde-b4e9-7b1beaa64b8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: f8a6759e-3a03-4c29-91aa-f6fc1e0cc7c7 kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj.18a8b40a57c3dfb1 namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 939dc3c1-f654-4216-8282-f64aa1a65384 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-56f57765b4 namespace: kserve-ci-e2e-test resourceVersion: "13376" uid: ae9699b6-e2ca-4cd6-a8f7-3cbd7b274383 kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-56f57765b4-c8xfj' metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: isvc-sklearn-graph-2-predictor-56f57765b4.18a8b3ec1dcd6d8f namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 84e7eba2-3e86-49ec-bb6d-3aebaab4c342 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 5c7eea82-bca2-4888-9fa6-2d81229369db kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-56f57765b4 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: isvc-sklearn-graph-2-predictor.18a8b3ec1aa7e704 namespace: kserve-ci-e2e-test resourceVersion: "13379" uid: ef92cd1b-5098-47d8-8f53-125d36a225a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13375" uid: 6a1b70d0-922d-46b1-b446-182a097b82d2 kind: Event lastTimestamp: "2026-04-22T14:25:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:24Z" name: isvc-sklearn-graph-2-predictor.18a8b3ef9945fa0b namespace: kserve-ci-e2e-test resourceVersion: "13853" uid: da4dba55-ec9c-4c8d-b65c-ca4efb4b12c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13375" uid: 6a1b70d0-922d-46b1-b446-182a097b82d2 kind: Event lastTimestamp: "2026-04-22T14:25:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:24Z" name: isvc-sklearn-graph-2-predictor.18a8b3ef99b4dc6b namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: 56d24ca9-4fd3-40e5-8e5f-7df466157d3e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13583" uid: 6a1b70d0-922d-46b1-b446-182a097b82d2 kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:39Z" name: isvc-sklearn-graph-2-predictor.18a8b3fa1710c45d namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: d21c3e8d-8821-4303-a9af-6140ffd9a739 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13583" uid: 6a1b70d0-922d-46b1-b446-182a097b82d2 kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:39Z" name: isvc-sklearn-graph-2-predictor.18a8b3fa1785746a namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: b0fc339e-1e06-4bac-a8c8-a20de7673626 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: 884e0752-db2d-400b-af89-f453adadca34 kind: Event lastTimestamp: "2026-04-22T14:26:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:18Z" name: isvc-sklearn-graph-2.18a8b3ec1a77879a namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: 4b463a35-2237-4d3f-b253-f3d99c4f821a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 884e0752-db2d-400b-af89-f453adadca34 kind: Event lastTimestamp: "2026-04-22T14:26:18Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-22T14:26:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:18Z" name: isvc-sklearn-graph-2.18a8b4031e6ca79d namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 541c903d-c31e-48bc-a6e6-c333e00fb877 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:24:39.582315Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13355" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-s8pc8 to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:24:39Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3ec174e5a6e namespace: kserve-ci-e2e-test resourceVersion: "13359" uid: 1ff27995-beca-4627-9506-f65e3b41f2e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:24:40Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3ec302c93d7 namespace: kserve-ci-e2e-test resourceVersion: "13405" uid: 0f397458-31fd-4d41-9aff-04b55019dc65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:24:40Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" metadata: creationTimestamp: "2026-04-22T14:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:40Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3ec31991da2 namespace: kserve-ci-e2e-test resourceVersion: "13407" uid: abdf90e4-09ed-4ec2-8c8e-0f16951a46e2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:24:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" in 13.182s (13.182s including waiting). Image size: 299663288 bytes.' metadata: creationTimestamp: "2026-04-22T14:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:53Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3ef43527c4e namespace: kserve-ci-e2e-test resourceVersion: "13523" uid: efb3b1a9-69e3-43d9-870d-ce2d00e6079c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:24:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:53Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3ef46531f76 namespace: kserve-ci-e2e-test resourceVersion: "13524" uid: 5eb38bf1-b999-45bb-839a-26f9e1c14bd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:24:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:53Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3ef46c10abd namespace: kserve-ci-e2e-test resourceVersion: "13525" uid: 4d0717da-5c00-4433-bb35-8cc30b01979d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:24:57Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-22T14:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:57Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3f03cee404a namespace: kserve-ci-e2e-test resourceVersion: "13606" uid: 360569a5-583a-440b-88f1-9fc33420e40f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:25:18Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.57s (21.57s including waiting). Image size: 1306326779 bytes.' metadata: creationTimestamp: "2026-04-22T14:25:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:18Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3f5429fadbe namespace: kserve-ci-e2e-test resourceVersion: "13799" uid: a92581a5-def4-4bc7-b83e-64eb284111a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:25:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:25:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:19Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3f54586b032 namespace: kserve-ci-e2e-test resourceVersion: "13800" uid: da6a4522-59f5-4f0d-9bb6-789712ee09ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:25:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:25:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:19Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3f54603b3f8 namespace: kserve-ci-e2e-test resourceVersion: "13801" uid: 63952368-6212-4b84-90d2-9274192684fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T14:25:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:26:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:25:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:50Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b3f56036728a namespace: kserve-ci-e2e-test resourceVersion: "14933" uid: 6ac7f9b3-56e3-492a-b67c-b4985e078845 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8 namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 458e541b-5615-4eef-813b-df55c095f0cf kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: isvc-xgboost-graph-predictor-784cb989b8-s8pc8.18a8b40a614797d2 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: e81eea47-a9f1-4c2b-bab4-14215c15d3ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "13347" uid: 1e91e258-94d0-402b-9b0f-a46be0b768d1 kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-s8pc8' metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a8b3ec16dd78df namespace: kserve-ci-e2e-test resourceVersion: "13356" uid: 784e9db2-9b53-4498-b3c2-c7ca3caeb3ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13345" uid: 8de09093-f466-4279-b164-8de56dd3fd64 kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: isvc-xgboost-graph-predictor.18a8b3ec141b3582 namespace: kserve-ci-e2e-test resourceVersion: "13349" uid: 6525bf37-d8ac-4ec1-93c6-58e5e47a8f37 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13346" uid: f31815a7-d33a-4555-977e-657c0543e607 kind: Event lastTimestamp: "2026-04-22T14:25:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:24Z" name: isvc-xgboost-graph-predictor.18a8b3ef92b78adc namespace: kserve-ci-e2e-test resourceVersion: "13851" uid: f8975df3-56a4-4916-a108-0c33cb460af5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13346" uid: f31815a7-d33a-4555-977e-657c0543e607 kind: Event lastTimestamp: "2026-04-22T14:25:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:24Z" name: isvc-xgboost-graph-predictor.18a8b3ef9323bcf9 namespace: kserve-ci-e2e-test resourceVersion: "13852" uid: 572a74d7-57ab-4e1f-97b8-92ef23706482 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13580" uid: f31815a7-d33a-4555-977e-657c0543e607 kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:39Z" name: isvc-xgboost-graph-predictor.18a8b3fa10a1ac96 namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 56ee7ffc-b848-41c5-8ca0-5a605dbec4f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13580" uid: f31815a7-d33a-4555-977e-657c0543e607 kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:39Z" name: isvc-xgboost-graph-predictor.18a8b3fa11150422 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 7817df0f-d29e-4a32-946e-ee839f43023d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13335" uid: d1172f1c-9fbc-4e42-af8e-5f77e67f6ebd kind: Event lastTimestamp: "2026-04-22T14:26:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:20Z" name: isvc-xgboost-graph.18a8b3ec140e5c98 namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 46688706-e6e0-48ba-bd89-897791ceec52 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: d1172f1c-9fbc-4e42-af8e-5f77e67f6ebd kind: Event lastTimestamp: "2026-04-22T14:26:20Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-22T14:26:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:20Z" name: isvc-xgboost-graph.18a8b40398c22277 namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 369982f1-164a-4083-a3a1-7ed8256b2c62 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:26:39.247869Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5ddf86697d-94lkl namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: af9ba33c-6129-461b-91bd-0459125543f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5ddf86697d-94lkl to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:26:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:26:39Z" name: model-chainer-5ddf86697d-94lkl.18a8b407f3edb64f namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: 7d4c7a9c-4a27-489d-aa56-e74eba030005 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:39Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5ddf86697d-94lkl namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: af9ba33c-6129-461b-91bd-0459125543f1 kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:39Z" name: model-chainer-5ddf86697d-94lkl.18a8b408016e427b namespace: kserve-ci-e2e-test resourceVersion: "14692" uid: ad86b556-163b-4806-b237-61107ecfb2a0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:40Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5ddf86697d-94lkl namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: af9ba33c-6129-461b-91bd-0459125543f1 kind: Event lastTimestamp: "2026-04-22T14:26:40Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:26:40Z" name: model-chainer-5ddf86697d-94lkl.18a8b40830297656 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: e00578eb-e059-4539-a71f-f823c471319e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5ddf86697d-94lkl namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: af9ba33c-6129-461b-91bd-0459125543f1 kind: Event lastTimestamp: "2026-04-22T14:26:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:40Z" name: model-chainer-5ddf86697d-94lkl.18a8b4083172bf45 namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: 28ea9d34-f832-41d9-b814-40c22778e4d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5ddf86697d-94lkl namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: af9ba33c-6129-461b-91bd-0459125543f1 kind: Event lastTimestamp: "2026-04-22T14:26:40Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-22T14:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:40Z" name: model-chainer-5ddf86697d-94lkl.18a8b40833e6a785 namespace: kserve-ci-e2e-test resourceVersion: "14717" uid: 14cc9aa2-989b-4dc5-ac26-ef3e41e378e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5ddf86697d-94lkl namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: af9ba33c-6129-461b-91bd-0459125543f1 kind: Event lastTimestamp: "2026-04-22T14:26:40Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-22T14:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:40Z" name: model-chainer-5ddf86697d-94lkl.18a8b408344119f8 namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: 771a2409-2037-488e-9380-9df0c826b11f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5ddf86697d-94lkl namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: af9ba33c-6129-461b-91bd-0459125543f1 kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: model-chainer-5ddf86697d-94lkl.18a8b40a504ca2a4 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 9bfc99d1-4351-45d2-9b68-abc28692b332 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5ddf86697d-94lkl namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: af9ba33c-6129-461b-91bd-0459125543f1 kind: Event lastTimestamp: "2026-04-22T14:27:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:27:16Z" name: model-chainer-5ddf86697d-94lkl.18a8b40ada10945e namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: e365b6f1-8490-449a-b41d-b00ab061d309 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5ddf86697d namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: f6f23655-66e6-4af2-a59a-1903fa728324 kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: 'Created pod: model-chainer-5ddf86697d-94lkl' metadata: creationTimestamp: "2026-04-22T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:39Z" name: model-chainer-5ddf86697d.18a8b407f37e3069 namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: 366c1ce2-f175-4ea1-950e-d19d485f5e3f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 3301e398-020f-4653-a899-c35758a15662 kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: Scaled up replica set model-chainer-5ddf86697d from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:39Z" name: model-chainer.18a8b407f2b45faf namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 75e92e81-b708-4a47-98b0-19e59352d903 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: 271eb8e1-c3d7-4da5-bb0e-642c4c0f97fe kind: Event lastTimestamp: "2026-04-22T14:26:39Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-22T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:39Z" name: model-chainer.18a8b407f7609c8a namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: 25ab05f7-94df-4d7f-ba9a-0614b4a18c15 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:38:44.762977Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0eb25-cb766f4b9-gvmml namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: 90cc6707-c4c6-42fd-8bf5-f5fbdfdead13 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0eb25-cb766f4b9-gvmml to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:38:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:38:44Z" name: sequence-graph-0eb25-cb766f4b9-gvmml.18a8b4b0dfffc495 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 139a6285-ef3f-4c70-a70e-51ee7e77501d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0eb25-cb766f4b9-gvmml namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: 90cc6707-c4c6-42fd-8bf5-f5fbdfdead13 kind: Event lastTimestamp: "2026-04-22T14:38:45Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:38:45Z" name: sequence-graph-0eb25-cb766f4b9-gvmml.18a8b4b0f8bba636 namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: ae4cbba8-fb87-42a3-847f-545f56db91a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0eb25} kind: Pod name: sequence-graph-0eb25-cb766f4b9-gvmml namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 90cc6707-c4c6-42fd-8bf5-f5fbdfdead13 kind: Event lastTimestamp: "2026-04-22T14:38:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:45Z" name: sequence-graph-0eb25-cb766f4b9-gvmml.18a8b4b0f9fb5f88 namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: cf38a9ec-d853-439d-9dfe-3e7a0ebdb248 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0eb25} kind: Pod name: sequence-graph-0eb25-cb766f4b9-gvmml namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 90cc6707-c4c6-42fd-8bf5-f5fbdfdead13 kind: Event lastTimestamp: "2026-04-22T14:38:45Z" message: 'Created container: sequence-graph-0eb25' metadata: creationTimestamp: "2026-04-22T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:45Z" name: sequence-graph-0eb25-cb766f4b9-gvmml.18a8b4b0fd201634 namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: a89119a3-623d-4028-a8bf-869c8727ca3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0eb25} kind: Pod name: sequence-graph-0eb25-cb766f4b9-gvmml namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 90cc6707-c4c6-42fd-8bf5-f5fbdfdead13 kind: Event lastTimestamp: "2026-04-22T14:38:45Z" message: Started container sequence-graph-0eb25 metadata: creationTimestamp: "2026-04-22T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:45Z" name: sequence-graph-0eb25-cb766f4b9-gvmml.18a8b4b0fd891810 namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: a6932e3d-9fd4-46b3-87b3-bbdf3a6b5ef9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0eb25} kind: Pod name: sequence-graph-0eb25-cb766f4b9-gvmml namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 90cc6707-c4c6-42fd-8bf5-f5fbdfdead13 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: Stopping container sequence-graph-0eb25 metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:59Z" name: sequence-graph-0eb25-cb766f4b9-gvmml.18a8b52414bf7ad0 namespace: kserve-ci-e2e-test resourceVersion: "24515" uid: b2c1caef-589f-4f0e-8768-c8c6d81b8ea5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0eb25} kind: Pod name: sequence-graph-0eb25-cb766f4b9-gvmml namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 90cc6707-c4c6-42fd-8bf5-f5fbdfdead13 kind: Event lastTimestamp: "2026-04-22T14:47:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:26Z" name: sequence-graph-0eb25-cb766f4b9-gvmml.18a8b52498a0e1bd namespace: kserve-ci-e2e-test resourceVersion: "24859" uid: 51d0ec84-07da-4cab-946c-7d687ec870e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0eb25-cb766f4b9 namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: e102fb85-f122-4029-a1ba-db73a375ebd7 kind: Event lastTimestamp: "2026-04-22T14:38:44Z" message: 'Created pod: sequence-graph-0eb25-cb766f4b9-gvmml' metadata: creationTimestamp: "2026-04-22T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:44Z" name: sequence-graph-0eb25-cb766f4b9.18a8b4b0df93ab42 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: 2ba16e11-687b-4956-a42a-04fa146724bd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 887f3a00-a127-4a28-a2ae-6a313997e1cb kind: Event lastTimestamp: "2026-04-22T14:38:44Z" message: Scaled up replica set sequence-graph-0eb25-cb766f4b9 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:44Z" name: sequence-graph-0eb25.18a8b4b0dec1c7d9 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: fb9552de-fb1a-43ba-8153-856a401de4ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: b4ebd001-f9ad-4331-87d7-8d281ef8faf3 kind: Event lastTimestamp: "2026-04-22T14:38:44Z" message: InferenceGraph [sequence-graph-0eb25] is Ready metadata: creationTimestamp: "2026-04-22T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:38:44Z" name: sequence-graph-0eb25.18a8b4b0e31e456d namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: 418b3405-4477-4352-8182-31a8ee841ea9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 324faf4c-adea-4eb5-bde9-3a0773ec67f2 kind: Event lastTimestamp: "2026-04-22T14:38:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:59Z" name: sequence-graph-0eb25.18a8b4b45d67e581 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 89b5d551-f6df-4c96-8218-f9ac750bb521 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 324faf4c-adea-4eb5-bde9-3a0773ec67f2 kind: Event lastTimestamp: "2026-04-22T14:38:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:59Z" name: sequence-graph-0eb25.18a8b4b45de2730a namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: a396d9a9-1b01-4753-9755-a83e145b9de5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:39:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: 324faf4c-adea-4eb5-bde9-3a0773ec67f2 kind: Event lastTimestamp: "2026-04-22T14:39:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:39:14Z" name: sequence-graph-0eb25.18a8b4b7dcb153ec namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: c7d49f05-b28d-4c55-9dee-4ef2627f9151 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:39:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: 324faf4c-adea-4eb5-bde9-3a0773ec67f2 kind: Event lastTimestamp: "2026-04-22T14:39:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:39:14Z" name: sequence-graph-0eb25.18a8b4b7dd319e24 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: e15a2b4b-2f3c-47b1-b5cf-73f7275a1797 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:27:59.572529Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1091a-65cf6f8f76-c76kt namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 38a61ddc-8d9b-4919-8f5c-6cdfa73cb8d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-1091a-65cf6f8f76-c76kt to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:27:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:27:59Z" name: sequence-graph-1091a-65cf6f8f76-c76kt.18a8b41aa7a6dab6 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: cb4cf808-e685-43c6-bf86-021b1cc07e55 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:59Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1091a-65cf6f8f76-c76kt namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: 38a61ddc-8d9b-4919-8f5c-6cdfa73cb8d5 kind: Event lastTimestamp: "2026-04-22T14:27:59Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:27:59Z" name: sequence-graph-1091a-65cf6f8f76-c76kt.18a8b41ac026332a namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: b3844028-9b26-4990-b2a9-ab06a8950203 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1091a} kind: Pod name: sequence-graph-1091a-65cf6f8f76-c76kt namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 38a61ddc-8d9b-4919-8f5c-6cdfa73cb8d5 kind: Event lastTimestamp: "2026-04-22T14:28:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:00Z" name: sequence-graph-1091a-65cf6f8f76-c76kt.18a8b41ac1c231c5 namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: a3322ef4-4067-40c3-8e16-948a059205ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1091a} kind: Pod name: sequence-graph-1091a-65cf6f8f76-c76kt namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 38a61ddc-8d9b-4919-8f5c-6cdfa73cb8d5 kind: Event lastTimestamp: "2026-04-22T14:28:00Z" message: 'Created container: sequence-graph-1091a' metadata: creationTimestamp: "2026-04-22T14:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:00Z" name: sequence-graph-1091a-65cf6f8f76-c76kt.18a8b41ac49a620c namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 5f21a90c-890c-4e9d-9832-37349d46d4bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1091a} kind: Pod name: sequence-graph-1091a-65cf6f8f76-c76kt namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 38a61ddc-8d9b-4919-8f5c-6cdfa73cb8d5 kind: Event lastTimestamp: "2026-04-22T14:28:00Z" message: Started container sequence-graph-1091a metadata: creationTimestamp: "2026-04-22T14:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:00Z" name: sequence-graph-1091a-65cf6f8f76-c76kt.18a8b41ac4f4b396 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: adc51484-737f-4fac-b2d6-62a83b2d7d3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1091a} kind: Pod name: sequence-graph-1091a-65cf6f8f76-c76kt namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 38a61ddc-8d9b-4919-8f5c-6cdfa73cb8d5 kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Stopping container sequence-graph-1091a metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:14Z" name: sequence-graph-1091a-65cf6f8f76-c76kt.18a8b48dd6d09d61 namespace: kserve-ci-e2e-test resourceVersion: "19155" uid: c2c18b57-92b5-48ed-8b82-8690ba2b8646 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:36:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1091a} kind: Pod name: sequence-graph-1091a-65cf6f8f76-c76kt namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 38a61ddc-8d9b-4919-8f5c-6cdfa73cb8d5 kind: Event lastTimestamp: "2026-04-22T14:36:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:36:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:41Z" name: sequence-graph-1091a-65cf6f8f76-c76kt.18a8b48e73f5f703 namespace: kserve-ci-e2e-test resourceVersion: "19504" uid: 1fe0ccc0-f838-4f46-bf39-62da888481af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-1091a-65cf6f8f76 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: b44eb343-aee3-4857-a4fa-c85ff94c4fed kind: Event lastTimestamp: "2026-04-22T14:27:59Z" message: 'Created pod: sequence-graph-1091a-65cf6f8f76-c76kt' metadata: creationTimestamp: "2026-04-22T14:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:59Z" name: sequence-graph-1091a-65cf6f8f76.18a8b41aa73a65ae namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: ad39a591-fcd9-45aa-a0a4-28390c58c438 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-1091a namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: fe01f9e3-7791-4df1-b2ea-4191b36c9867 kind: Event lastTimestamp: "2026-04-22T14:27:59Z" message: Scaled up replica set sequence-graph-1091a-65cf6f8f76 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:59Z" name: sequence-graph-1091a.18a8b41aa58967f0 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 3898bfb7-ab3f-496b-8021-839023e95842 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-1091a namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: 22eb99f0-38c0-4500-a2cd-285082ec0667 kind: Event lastTimestamp: "2026-04-22T14:27:59Z" message: InferenceGraph [sequence-graph-1091a] is Ready metadata: creationTimestamp: "2026-04-22T14:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:59Z" name: sequence-graph-1091a.18a8b41aaaf473d2 namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 53307d79-930f-463a-b500-04410f799585 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:28:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1091a namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: bbbe81d4-2dc2-4061-bb0d-e14d4a976ec8 kind: Event lastTimestamp: "2026-04-22T14:28:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:29Z" name: sequence-graph-1091a.18a8b41e2414dec4 namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: 68664506-8a77-45ce-8ea7-b8a8abecd1c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:28:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1091a namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: bbbe81d4-2dc2-4061-bb0d-e14d4a976ec8 kind: Event lastTimestamp: "2026-04-22T14:28:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:28:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:29Z" name: sequence-graph-1091a.18a8b41e247f906b namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: d6250da9-606b-44d9-a6b8-9cd63808943e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:37:24.463941Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-33e8e-5c55857f76-xpths namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: 96a41e80-8086-468e-a98d-7444e1af730a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-33e8e-5c55857f76-xpths to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:37:24Z" name: sequence-graph-33e8e-5c55857f76-xpths.18a8b49e2dcdbd1e namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: f43d9ec0-9602-4500-9dd1-cbdce49ccf61 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:24Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-33e8e-5c55857f76-xpths namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: 96a41e80-8086-468e-a98d-7444e1af730a kind: Event lastTimestamp: "2026-04-22T14:37:24Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:37:24Z" name: sequence-graph-33e8e-5c55857f76-xpths.18a8b49e467199a9 namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 1ff8cd34-6040-4101-b202-329ade1b274f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-33e8e} kind: Pod name: sequence-graph-33e8e-5c55857f76-xpths namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 96a41e80-8086-468e-a98d-7444e1af730a kind: Event lastTimestamp: "2026-04-22T14:37:24Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:24Z" name: sequence-graph-33e8e-5c55857f76-xpths.18a8b49e47bf3308 namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: de89c2f3-7ec1-4107-bb99-b2f4d5cb0825 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-33e8e} kind: Pod name: sequence-graph-33e8e-5c55857f76-xpths namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 96a41e80-8086-468e-a98d-7444e1af730a kind: Event lastTimestamp: "2026-04-22T14:37:24Z" message: 'Created container: sequence-graph-33e8e' metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:24Z" name: sequence-graph-33e8e-5c55857f76-xpths.18a8b49e4abb6abf namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: 1e4602f3-a62f-46be-b4b7-56bdeafdd41e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-33e8e} kind: Pod name: sequence-graph-33e8e-5c55857f76-xpths namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 96a41e80-8086-468e-a98d-7444e1af730a kind: Event lastTimestamp: "2026-04-22T14:37:24Z" message: Started container sequence-graph-33e8e metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:24Z" name: sequence-graph-33e8e-5c55857f76-xpths.18a8b49e4b3bd79b namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: e6de1ab9-a279-45bd-8681-b1e07509dac4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-33e8e} kind: Pod name: sequence-graph-33e8e-5c55857f76-xpths namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 96a41e80-8086-468e-a98d-7444e1af730a kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Stopping container sequence-graph-33e8e metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:34Z" name: sequence-graph-33e8e-5c55857f76-xpths.18a8b4a086a7740e namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: 9869b213-ea6c-4910-b214-7f53eb59e70b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-33e8e} kind: Pod name: sequence-graph-33e8e-5c55857f76-xpths namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 96a41e80-8086-468e-a98d-7444e1af730a kind: Event lastTimestamp: "2026-04-22T14:38:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:01Z" name: sequence-graph-33e8e-5c55857f76-xpths.18a8b4a0fe008822 namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: 9047d03b-b61c-4193-abb8-d30fb02f9d5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-33e8e-5c55857f76 namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 62f871d8-f10a-464d-9658-26f3029b4f4e kind: Event lastTimestamp: "2026-04-22T14:37:24Z" message: 'Created pod: sequence-graph-33e8e-5c55857f76-xpths' metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:24Z" name: sequence-graph-33e8e-5c55857f76.18a8b49e2d6b48af namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 07e5eedd-cbfe-429e-a919-10ee2f8c2499 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-33e8e namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: d0ab1e74-ab45-466d-b4bc-f95389cbd9ea kind: Event lastTimestamp: "2026-04-22T14:37:24Z" message: Scaled up replica set sequence-graph-33e8e-5c55857f76 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:24Z" name: sequence-graph-33e8e.18a8b49e2c90b8ab namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: ee397047-1dc9-4914-9f17-f8c251022511 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-33e8e namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: bc1df9f1-2780-4a1f-9455-c0f5a88c37cc kind: Event lastTimestamp: "2026-04-22T14:37:24Z" message: InferenceGraph [sequence-graph-33e8e] is Ready metadata: creationTimestamp: "2026-04-22T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:24Z" name: sequence-graph-33e8e.18a8b49e32391abe namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: 455c19b1-f800-4eda-bec4-836a086612e0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:48:50.493378Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-055dc-576fcf5468-jgbgv namespace: kserve-ci-e2e-test resourceVersion: "25775" uid: 976866d0-ceea-428e-b669-8b5641eb6c17 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-055dc-576fcf5468-jgbgv to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:48:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:48:50Z" name: splitter-graph-055dc-576fcf5468-jgbgv.18a8b53de85839af namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: 6e085c9c-223f-4684-b0b8-bdeb7b94fcf5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:50Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-055dc-576fcf5468-jgbgv namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: 976866d0-ceea-428e-b669-8b5641eb6c17 kind: Event lastTimestamp: "2026-04-22T14:48:50Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:48:50Z" name: splitter-graph-055dc-576fcf5468-jgbgv.18a8b53e01137c56 namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: 602c6336-4d83-403d-b5cd-5abcbd5e0032 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-055dc} kind: Pod name: splitter-graph-055dc-576fcf5468-jgbgv namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: 976866d0-ceea-428e-b669-8b5641eb6c17 kind: Event lastTimestamp: "2026-04-22T14:48:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:50Z" name: splitter-graph-055dc-576fcf5468-jgbgv.18a8b53e02e4fa8e namespace: kserve-ci-e2e-test resourceVersion: "25795" uid: 6f7d4964-2893-4466-9a4f-1e8c10797aff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-055dc} kind: Pod name: splitter-graph-055dc-576fcf5468-jgbgv namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: 976866d0-ceea-428e-b669-8b5641eb6c17 kind: Event lastTimestamp: "2026-04-22T14:48:50Z" message: 'Created container: splitter-graph-055dc' metadata: creationTimestamp: "2026-04-22T14:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:50Z" name: splitter-graph-055dc-576fcf5468-jgbgv.18a8b53e058d8e79 namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: 276bee5c-ef42-49f8-b06e-7a384a429083 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-055dc} kind: Pod name: splitter-graph-055dc-576fcf5468-jgbgv namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: 976866d0-ceea-428e-b669-8b5641eb6c17 kind: Event lastTimestamp: "2026-04-22T14:48:50Z" message: Started container splitter-graph-055dc metadata: creationTimestamp: "2026-04-22T14:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:51Z" name: splitter-graph-055dc-576fcf5468-jgbgv.18a8b53e060e0c09 namespace: kserve-ci-e2e-test resourceVersion: "25797" uid: 177986ad-f786-47a0-bfa0-1682ff03414e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:57:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-055dc} kind: Pod name: splitter-graph-055dc-576fcf5468-jgbgv namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: 976866d0-ceea-428e-b669-8b5641eb6c17 kind: Event lastTimestamp: "2026-04-22T14:57:04Z" message: Stopping container splitter-graph-055dc metadata: creationTimestamp: "2026-04-22T14:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:57:04Z" name: splitter-graph-055dc-576fcf5468-jgbgv.18a8b5b106c80b21 namespace: kserve-ci-e2e-test resourceVersion: "29540" uid: 85ec1334-b51a-4c7f-8965-80c598a05cc0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-055dc} kind: Pod name: splitter-graph-055dc-576fcf5468-jgbgv namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: 976866d0-ceea-428e-b669-8b5641eb6c17 kind: Event lastTimestamp: "2026-04-22T14:57:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:57:32Z" name: splitter-graph-055dc-576fcf5468-jgbgv.18a8b5b19fdcf9c5 namespace: kserve-ci-e2e-test resourceVersion: "29772" uid: 02bec28b-489a-427d-922c-f8cb2d613dc6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-055dc-576fcf5468 namespace: kserve-ci-e2e-test resourceVersion: "25771" uid: a2071b33-ec2c-453d-97d2-ade0ebae2e72 kind: Event lastTimestamp: "2026-04-22T14:48:50Z" message: 'Created pod: splitter-graph-055dc-576fcf5468-jgbgv' metadata: creationTimestamp: "2026-04-22T14:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:50Z" name: splitter-graph-055dc-576fcf5468.18a8b53de72f13e3 namespace: kserve-ci-e2e-test resourceVersion: "25777" uid: 8ea04d3f-cf8e-447b-bf8e-4be5c178f15e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-055dc namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: 804fd058-e44e-4888-9a87-74c529eefa00 kind: Event lastTimestamp: "2026-04-22T14:48:50Z" message: Scaled up replica set splitter-graph-055dc-576fcf5468 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:50Z" name: splitter-graph-055dc.18a8b53de5a4e1f7 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: b126320a-4c93-4b76-896e-21af6665963c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-055dc namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: 6df148e4-5fad-40ff-9825-5ae036a510cf kind: Event lastTimestamp: "2026-04-22T14:48:50Z" message: InferenceGraph [splitter-graph-055dc] is Ready metadata: creationTimestamp: "2026-04-22T14:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:48:50Z" name: splitter-graph-055dc.18a8b53defa864ed namespace: kserve-ci-e2e-test resourceVersion: "25790" uid: eff056af-5fb1-469f-a9b3-3ccf5128a3e0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:49:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-055dc namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: 9576d3cb-4aba-4b4f-99e0-714d801626d0 kind: Event lastTimestamp: "2026-04-22T14:49:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:49:20Z" name: splitter-graph-055dc.18a8b5416512db2b namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: d989a8cc-fb1b-48be-819c-b35f3052a0b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:49:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-055dc namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: 9576d3cb-4aba-4b4f-99e0-714d801626d0 kind: Event lastTimestamp: "2026-04-22T14:49:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:49:20Z" name: splitter-graph-055dc.18a8b541659de4c2 namespace: kserve-ci-e2e-test resourceVersion: "26304" uid: 6030d6b5-940f-45bc-8e99-7f2004c65452 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:47:30.047759Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f4200-749f97c4cf-6f7k6 namespace: kserve-ci-e2e-test resourceVersion: "24900" uid: d6dbfab9-6aed-49fc-81a5-abfd0cfa6851 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-f4200-749f97c4cf-6f7k6 to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:47:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:47:30Z" name: splitter-graph-f4200-749f97c4cf-6f7k6.18a8b52b2d697d6d namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 022d09b4-c50d-4142-bb39-872b40f86824 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:30Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f4200-749f97c4cf-6f7k6 namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: d6dbfab9-6aed-49fc-81a5-abfd0cfa6851 kind: Event lastTimestamp: "2026-04-22T14:47:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-f4200-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T14:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:30Z" name: splitter-graph-f4200-749f97c4cf-6f7k6.18a8b52b376956a5 namespace: kserve-ci-e2e-test resourceVersion: "24919" uid: 0d4c2a0a-b5d4-48db-874e-52d5e50d9204 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:31Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f4200-749f97c4cf-6f7k6 namespace: kserve-ci-e2e-test resourceVersion: "24910" uid: d6dbfab9-6aed-49fc-81a5-abfd0cfa6851 kind: Event lastTimestamp: "2026-04-22T14:47:31Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:47:31Z" name: splitter-graph-f4200-749f97c4cf-6f7k6.18a8b52b69ccfdc3 namespace: kserve-ci-e2e-test resourceVersion: "24932" uid: 529c3e91-68ab-41e5-a05e-69f4ee942eb2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f4200} kind: Pod name: splitter-graph-f4200-749f97c4cf-6f7k6 namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: d6dbfab9-6aed-49fc-81a5-abfd0cfa6851 kind: Event lastTimestamp: "2026-04-22T14:47:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:31Z" name: splitter-graph-f4200-749f97c4cf-6f7k6.18a8b52b6b156061 namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: a5b267d0-e823-4b2b-a7d4-8f978ec911ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f4200} kind: Pod name: splitter-graph-f4200-749f97c4cf-6f7k6 namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: d6dbfab9-6aed-49fc-81a5-abfd0cfa6851 kind: Event lastTimestamp: "2026-04-22T14:47:31Z" message: 'Created container: splitter-graph-f4200' metadata: creationTimestamp: "2026-04-22T14:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:31Z" name: splitter-graph-f4200-749f97c4cf-6f7k6.18a8b52b6dd6c49f namespace: kserve-ci-e2e-test resourceVersion: "24936" uid: e6f13511-7d1b-40d6-967e-49849a0c92f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f4200} kind: Pod name: splitter-graph-f4200-749f97c4cf-6f7k6 namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: d6dbfab9-6aed-49fc-81a5-abfd0cfa6851 kind: Event lastTimestamp: "2026-04-22T14:47:31Z" message: Started container splitter-graph-f4200 metadata: creationTimestamp: "2026-04-22T14:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:31Z" name: splitter-graph-f4200-749f97c4cf-6f7k6.18a8b52b6e425197 namespace: kserve-ci-e2e-test resourceVersion: "24937" uid: 7f8d6461-5968-4365-8048-a8dcb6426f7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f4200} kind: Pod name: splitter-graph-f4200-749f97c4cf-6f7k6 namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: d6dbfab9-6aed-49fc-81a5-abfd0cfa6851 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Stopping container splitter-graph-f4200 metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:40Z" name: splitter-graph-f4200-749f97c4cf-6f7k6.18a8b52d8a3c8a62 namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: ff1dfc7c-1eb9-4f29-a8be-a70545660e66 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f4200} kind: Pod name: splitter-graph-f4200-749f97c4cf-6f7k6 namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: d6dbfab9-6aed-49fc-81a5-abfd0cfa6851 kind: Event lastTimestamp: "2026-04-22T14:48:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:07Z" name: splitter-graph-f4200-749f97c4cf-6f7k6.18a8b52e0504d6a7 namespace: kserve-ci-e2e-test resourceVersion: "25371" uid: 33eb3cd5-b24f-43a5-a2f9-23315c4f70c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-f4200-749f97c4cf namespace: kserve-ci-e2e-test resourceVersion: "24897" uid: 63fc57cc-6b11-487d-8511-b86cbcf74598 kind: Event lastTimestamp: "2026-04-22T14:47:30Z" message: 'Created pod: splitter-graph-f4200-749f97c4cf-6f7k6' metadata: creationTimestamp: "2026-04-22T14:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:30Z" name: splitter-graph-f4200-749f97c4cf.18a8b52b2c884210 namespace: kserve-ci-e2e-test resourceVersion: "24901" uid: a077b0be-1601-4891-a9c2-96cdaacd04f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-f4200 namespace: kserve-ci-e2e-test resourceVersion: "24895" uid: e05e6f45-5bb9-4cfe-a41e-a765ad116479 kind: Event lastTimestamp: "2026-04-22T14:47:30Z" message: Scaled up replica set splitter-graph-f4200-749f97c4cf from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:30Z" name: splitter-graph-f4200.18a8b52b2bb5ae70 namespace: kserve-ci-e2e-test resourceVersion: "24898" uid: cbfac475-b556-48c1-a4fa-926351cd2d5d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-f4200 namespace: kserve-ci-e2e-test resourceVersion: "24912" uid: 441c3209-39e6-441e-9334-0bb033ee08e5 kind: Event lastTimestamp: "2026-04-22T14:47:30Z" message: InferenceGraph [splitter-graph-f4200] is Ready metadata: creationTimestamp: "2026-04-22T14:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:30Z" name: splitter-graph-f4200.18a8b52b30dce415 namespace: kserve-ci-e2e-test resourceVersion: "24915" uid: 7a835155-305d-4556-8774-c0ead500ff03 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-f4200 namespace: kserve-ci-e2e-test resourceVersion: "24887" uid: 441c3209-39e6-441e-9334-0bb033ee08e5 kind: Event lastTimestamp: "2026-04-22T14:47:30Z" message: 'Failed to update status for InferenceGraph "splitter-graph-f4200": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-f4200": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:30Z" name: splitter-graph-f4200.18a8b52b336802c0 namespace: kserve-ci-e2e-test resourceVersion: "24916" uid: f314642d-466f-45f8-bfc7-8d6d78035482 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-f4200 namespace: kserve-ci-e2e-test resourceVersion: "24887" uid: 441c3209-39e6-441e-9334-0bb033ee08e5 kind: Event lastTimestamp: "2026-04-22T14:47:30Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-f4200": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:30Z" name: splitter-graph-f4200.18a8b52b33687711 namespace: kserve-ci-e2e-test resourceVersion: "24917" uid: 8ebdb941-5892-4bbc-aecd-bfba138f1f7e reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:47:40.321204Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r namespace: kserve-ci-e2e-test resourceVersion: "25045" uid: a8230180-72c1-4546-a36a-6c222a313b31 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-055dc-predictor-757d8d68d-tlq9r to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r.18a8b52d91c1cf79 namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: 285f7759-732e-451d-9acc-90ff53e91b75 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r namespace: kserve-ci-e2e-test resourceVersion: "25059" uid: a8230180-72c1-4546-a36a-6c222a313b31 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r.18a8b52d9be46b25 namespace: kserve-ci-e2e-test resourceVersion: "25109" uid: fa8fff27-abfb-40c0-8c11-dac73e4423f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: a8230180-72c1-4546-a36a-6c222a313b31 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r.18a8b52d9d34b420 namespace: kserve-ci-e2e-test resourceVersion: "25111" uid: f9af8a7a-7df1-42ca-80ee-84f620fbdf0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: a8230180-72c1-4546-a36a-6c222a313b31 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r.18a8b52da561d782 namespace: kserve-ci-e2e-test resourceVersion: "25120" uid: 4469db71-43b3-4b3e-89d1-5dee147ae385 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: a8230180-72c1-4546-a36a-6c222a313b31 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r.18a8b52da5e02096 namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: ebb64aa8-fab3-481b-8f3d-9fc52b8fcd49 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: a8230180-72c1-4546-a36a-6c222a313b31 kind: Event lastTimestamp: "2026-04-22T14:48:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:21Z" name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r.18a8b52db389e60c namespace: kserve-ci-e2e-test resourceVersion: "25532" uid: d4cd6abe-cfe3-4a26-98fa-59d0c18bd673 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: a8230180-72c1-4546-a36a-6c222a313b31 kind: Event lastTimestamp: "2026-04-22T14:57:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:57:05Z" name: success-200-isvc-055dc-predictor-757d8d68d-tlq9r.18a8b5b10d517cbf namespace: kserve-ci-e2e-test resourceVersion: "29555" uid: d63bfd88-55a3-4374-a5b5-4de0b24fad96 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-055dc-predictor-757d8d68d namespace: kserve-ci-e2e-test resourceVersion: "25038" uid: 16875b9d-ae90-4f81-8b4f-525aee4ebd47 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: 'Created pod: success-200-isvc-055dc-predictor-757d8d68d-tlq9r' metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-055dc-predictor-757d8d68d.18a8b52d91452ae7 namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: c49428a5-63fc-4215-8969-33c82f04e09d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: 7056f950-8d84-4f79-b69d-aa1c264679c4 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Scaled up replica set success-200-isvc-055dc-predictor-757d8d68d from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-055dc-predictor.18a8b52d8fa2ab6e namespace: kserve-ci-e2e-test resourceVersion: "25040" uid: f23b401d-42b4-4d99-8162-66fc5f13a0f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:47:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: fbe1abeb-d753-4045-a398-9371d7894ee9 kind: Event lastTimestamp: "2026-04-22T14:48:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:47:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:10Z" name: success-200-isvc-055dc-predictor.18a8b5310e97caee namespace: kserve-ci-e2e-test resourceVersion: "25421" uid: 7b6e7e99-583d-4ac1-90fa-b09892eb3806 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:47:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: fbe1abeb-d753-4045-a398-9371d7894ee9 kind: Event lastTimestamp: "2026-04-22T14:48:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:47:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:10Z" name: success-200-isvc-055dc-predictor.18a8b5310ef89d35 namespace: kserve-ci-e2e-test resourceVersion: "25422" uid: aafee208-a404-4393-96b3-ae10ad8a1a10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:48:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: fbe1abeb-d753-4045-a398-9371d7894ee9 kind: Event lastTimestamp: "2026-04-22T14:48:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:55Z" name: success-200-isvc-055dc-predictor.18a8b5380cd460de namespace: kserve-ci-e2e-test resourceVersion: "25833" uid: b12e7d9b-6a84-4db8-a89f-9cc60e88e8ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:48:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-055dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: fbe1abeb-d753-4045-a398-9371d7894ee9 kind: Event lastTimestamp: "2026-04-22T14:48:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:55Z" name: success-200-isvc-055dc-predictor.18a8b5380d4699a2 namespace: kserve-ci-e2e-test resourceVersion: "25834" uid: 0684676f-e5dd-4e05-8409-07d7b79b936d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-055dc namespace: kserve-ci-e2e-test resourceVersion: "25028" uid: daff81fd-b3f0-48d9-b1a0-9f3e665fe362 kind: Event lastTimestamp: "2026-04-22T14:48:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:48:31Z" name: success-200-isvc-055dc.18a8b52d8f762701 namespace: kserve-ci-e2e-test resourceVersion: "25618" uid: c2983a20-1dc6-44f5-8e1b-4bcf85b5477c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-055dc namespace: kserve-ci-e2e-test resourceVersion: "25028" uid: daff81fd-b3f0-48d9-b1a0-9f3e665fe362 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: 'Failed to update status for InferenceService "success-200-isvc-055dc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-055dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-055dc.18a8b52d8fcc2d23 namespace: kserve-ci-e2e-test resourceVersion: "25042" uid: 4e84e341-8491-483c-8888-5106d62c8931 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-055dc namespace: kserve-ci-e2e-test resourceVersion: "25028" uid: daff81fd-b3f0-48d9-b1a0-9f3e665fe362 kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-055dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-055dc.18a8b52d8fcc8e6e namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 8f26cd83-cf6e-4e0a-a380-8c92cc97ad5a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-055dc namespace: kserve-ci-e2e-test resourceVersion: "25615" uid: daff81fd-b3f0-48d9-b1a0-9f3e665fe362 kind: Event lastTimestamp: "2026-04-22T14:48:31Z" message: InferenceService [success-200-isvc-055dc] is Ready metadata: creationTimestamp: "2026-04-22T14:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:48:31Z" name: success-200-isvc-055dc.18a8b539975212d7 namespace: kserve-ci-e2e-test resourceVersion: "25616" uid: 67826669-8276-4f83-b10a-12515472812c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:37:34.665891Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5 namespace: kserve-ci-e2e-test resourceVersion: "20207" uid: 8774659b-7dd0-4ab9-9cce-0e1114f33e93 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5 to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5.18a8b4a08de31773 namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 8e01d2bc-7f1b-4016-ad3f-f1cb3af119b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5 namespace: kserve-ci-e2e-test resourceVersion: "20226" uid: 8774659b-7dd0-4ab9-9cce-0e1114f33e93 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5.18a8b4a097557394 namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 33b78af1-a0f7-4635-ae9e-e596b891e5c3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 8774659b-7dd0-4ab9-9cce-0e1114f33e93 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5.18a8b4a098d4e4f3 namespace: kserve-ci-e2e-test resourceVersion: "20260" uid: f0203ab9-46dd-4744-81cc-ebb5059b3fec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 8774659b-7dd0-4ab9-9cce-0e1114f33e93 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5.18a8b4a09eb6162d namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: 74100a80-550c-4cbc-8be1-4351b330d9cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 8774659b-7dd0-4ab9-9cce-0e1114f33e93 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5.18a8b4a09f320cd9 namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: a9e75b27-3bda-4d5e-bc31-d07393f2003b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 8774659b-7dd0-4ab9-9cce-0e1114f33e93 kind: Event lastTimestamp: "2026-04-22T14:38:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:16Z" name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5.18a8b4a0d72cdbb5 namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 96acac00-adab-434c-b076-2fd4090309d9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 8774659b-7dd0-4ab9-9cce-0e1114f33e93 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:59Z" name: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5.18a8b52422cbd557 namespace: kserve-ci-e2e-test resourceVersion: "24577" uid: 528cdb23-7946-4dcb-b872-8aedb8ef22ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0eb25-predictor-bf4c6c446 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 3bb54f49-1ec6-4170-b37d-c9683ceaa053 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: 'Created pod: success-200-isvc-0eb25-predictor-bf4c6c446-wvzg5' metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-0eb25-predictor-bf4c6c446.18a8b4a08d581176 namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: b114f369-23a1-4117-a287-838d1658792f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0eb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: edeb3f66-b0e5-473e-a8c8-434c314350bf kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Scaled up replica set success-200-isvc-0eb25-predictor-bf4c6c446 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-0eb25-predictor.18a8b4a08bdb9468 namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: a0cfcae1-3a23-4aee-9ac8-fa72be6fa029 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0eb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: 829ebd53-1a38-48bc-b8f9-8ace5ac5c6e5 kind: Event lastTimestamp: "2026-04-22T14:38:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:19Z" name: success-200-isvc-0eb25-predictor.18a8b4a40a4253ce namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: f6ecd3f2-5aed-4fc0-be73-89ccfb111b05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0eb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: 829ebd53-1a38-48bc-b8f9-8ace5ac5c6e5 kind: Event lastTimestamp: "2026-04-22T14:38:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:19Z" name: success-200-isvc-0eb25-predictor.18a8b4a40a8532d4 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1343d4d2-b0be-4922-90da-d54301ec161c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:38:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0eb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: 829ebd53-1a38-48bc-b8f9-8ace5ac5c6e5 kind: Event lastTimestamp: "2026-04-22T14:39:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:39:04Z" name: success-200-isvc-0eb25-predictor.18a8b4ae8699840d namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: ac85d217-8f97-457e-943c-16e4f2136afd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:38:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0eb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: 829ebd53-1a38-48bc-b8f9-8ace5ac5c6e5 kind: Event lastTimestamp: "2026-04-22T14:39:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:39:04Z" name: success-200-isvc-0eb25-predictor.18a8b4ae86e113c8 namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: 44fc6860-bfc7-439b-b106-f468858e3e89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20178" uid: d3775178-6de3-4a91-831a-0cec2e5ff2ab kind: Event lastTimestamp: "2026-04-22T14:38:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:38:26Z" name: success-200-isvc-0eb25.18a8b4a08bb00b53 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: ed1f95eb-0cd9-469b-add8-17d47fa7bc4a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20178" uid: d3775178-6de3-4a91-831a-0cec2e5ff2ab kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: 'Failed to update status for InferenceService "success-200-isvc-0eb25": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0eb25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-0eb25.18a8b4a08c0a2e81 namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 56435e24-b2c1-469c-a36e-16fb7adca81b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20178" uid: d3775178-6de3-4a91-831a-0cec2e5ff2ab kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0eb25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-0eb25.18a8b4a08c0a9a4d namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: 6979a1e8-7bcd-446b-b231-ca8d4620d62d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0eb25 namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: d3775178-6de3-4a91-831a-0cec2e5ff2ab kind: Event lastTimestamp: "2026-04-22T14:38:26Z" message: InferenceService [success-200-isvc-0eb25] is Ready metadata: creationTimestamp: "2026-04-22T14:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:38:26Z" name: success-200-isvc-0eb25.18a8b4acbabb8429 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: c6b8fdd5-b571-4536-8736-fd33c44bddcd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:26:49.538819Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1091a-predictor-579b887b74-bdjvv namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 71c4c9fd-84c0-47d4-ab0a-204be014b8d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1091a-predictor-579b887b74-bdjvv to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:26:49Z" name: success-200-isvc-1091a-predictor-579b887b74-bdjvv.18a8b40a595129d6 namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 5c0a0a8a-f638-4ca2-aa7a-8fde56b99035 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1091a-predictor-579b887b74-bdjvv namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 71c4c9fd-84c0-47d4-ab0a-204be014b8d0 kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:26:49Z" name: success-200-isvc-1091a-predictor-579b887b74-bdjvv.18a8b40a63f98cb6 namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 27f3cff0-b8ef-4c5d-8c76-b43366c9268f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1091a-predictor-579b887b74-bdjvv namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 71c4c9fd-84c0-47d4-ab0a-204be014b8d0 kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: success-200-isvc-1091a-predictor-579b887b74-bdjvv.18a8b40a657504a3 namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: abff5d92-e09b-4b9c-9043-43be13b8ce48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1091a-predictor-579b887b74-bdjvv namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 71c4c9fd-84c0-47d4-ab0a-204be014b8d0 kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: success-200-isvc-1091a-predictor-579b887b74-bdjvv.18a8b40a6be1bb38 namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 8151c050-48f8-4a55-8c06-ae11441472c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1091a-predictor-579b887b74-bdjvv namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 71c4c9fd-84c0-47d4-ab0a-204be014b8d0 kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:49Z" name: success-200-isvc-1091a-predictor-579b887b74-bdjvv.18a8b40a6c68097a namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 8fecb5d2-f23f-4dde-a1d2-177a1eedc000 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1091a-predictor-579b887b74-bdjvv namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 71c4c9fd-84c0-47d4-ab0a-204be014b8d0 kind: Event lastTimestamp: "2026-04-22T14:27:31Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:27:31Z" name: success-200-isvc-1091a-predictor-579b887b74-bdjvv.18a8b40aa22e0fe8 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: e4b066c5-ae3d-405e-9a23-9eb1085c23bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1091a-predictor-579b887b74-bdjvv namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 71c4c9fd-84c0-47d4-ab0a-204be014b8d0 kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-1091a-predictor-579b887b74-bdjvv.18a8b48dddd64cd3 namespace: kserve-ci-e2e-test resourceVersion: "19199" uid: 23e2f658-ec8c-4e11-a5c2-7d0de15fdb9a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1091a-predictor-579b887b74 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 1a992916-b0c4-4d56-9758-8f65141ddf6a kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: 'Created pod: success-200-isvc-1091a-predictor-579b887b74-bdjvv' metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:49Z" name: success-200-isvc-1091a-predictor-579b887b74.18a8b40a58dea9f1 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: 78c90c11-7bb3-4011-900f-a8d5292544fb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: 405938fe-0ca3-4f22-8521-8c86da17a299 kind: Event lastTimestamp: "2026-04-22T14:26:49Z" message: Scaled up replica set success-200-isvc-1091a-predictor-579b887b74 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:49Z" name: success-200-isvc-1091a-predictor.18a8b40a5731091b namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: c5533a75-cf1d-4d6f-ad14-da77d7fc2622 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: ca14f787-4419-4418-bd1c-3f70b918ca53 kind: Event lastTimestamp: "2026-04-22T14:27:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:04Z" name: success-200-isvc-1091a-predictor.18a8b40dd54626f0 namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: 0a215f51-c431-4027-ad77-87885046c60f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: ca14f787-4419-4418-bd1c-3f70b918ca53 kind: Event lastTimestamp: "2026-04-22T14:27:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:04Z" name: success-200-isvc-1091a-predictor.18a8b40dd5c6e92e namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 1be94a98-d878-4205-b436-bc818b5af58d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:27:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15080" uid: ca14f787-4419-4418-bd1c-3f70b918ca53 kind: Event lastTimestamp: "2026-04-22T14:27:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:49Z" name: success-200-isvc-1091a-predictor.18a8b4115486c91b namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: c42ad224-f77c-485e-b3db-5e2a33ab0a81 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:27:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1091a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15080" uid: ca14f787-4419-4418-bd1c-3f70b918ca53 kind: Event lastTimestamp: "2026-04-22T14:27:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:49Z" name: success-200-isvc-1091a-predictor.18a8b41154f68d13 namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: e933fe52-aaa3-41a4-82ff-9c38d0c19214 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1091a namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: eb0e4233-d10d-43f7-99ef-0b219a024191 kind: Event lastTimestamp: "2026-04-22T14:27:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:41Z" name: success-200-isvc-1091a.18a8b40a568f605b namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: 98c02575-6364-444b-b849-218247275e56 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1091a namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: eb0e4233-d10d-43f7-99ef-0b219a024191 kind: Event lastTimestamp: "2026-04-22T14:27:41Z" message: InferenceService [success-200-isvc-1091a] is Ready metadata: creationTimestamp: "2026-04-22T14:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:41Z" name: success-200-isvc-1091a.18a8b41686cca772 namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 01d49cd9-31c5-4958-876a-1aeb44562cbc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:24:39.170065Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw namespace: kserve-ci-e2e-test resourceVersion: "13269" uid: b9912495-2ac2-4d13-9bb0-3bcced8805ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1cfdf-predictor-6985558666-ml6vw to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:24:39Z" name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw.18a8b3ebfebbee36 namespace: kserve-ci-e2e-test resourceVersion: "13276" uid: 274ea773-52b9-441e-ba2b-89fedbe76753 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw namespace: kserve-ci-e2e-test resourceVersion: "13281" uid: b9912495-2ac2-4d13-9bb0-3bcced8805ba kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:24:39Z" name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw.18a8b3ec06e59bcc namespace: kserve-ci-e2e-test resourceVersion: "13287" uid: 63d2160e-d2c9-4b81-ba38-e14166c39a66 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw namespace: kserve-ci-e2e-test resourceVersion: "13272" uid: b9912495-2ac2-4d13-9bb0-3bcced8805ba kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:39Z" name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw.18a8b3ec0833339b namespace: kserve-ci-e2e-test resourceVersion: "13292" uid: 21aa24c5-8ed5-418b-b842-3eaf56c340b2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw namespace: kserve-ci-e2e-test resourceVersion: "13272" uid: b9912495-2ac2-4d13-9bb0-3bcced8805ba kind: Event lastTimestamp: "2026-04-22T14:24:54Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" in 14.73s (14.73s including waiting). Image size: 1334121572 bytes.' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:54Z" name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw.18a8b3ef7633fff4 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: d62def38-3c03-4364-9f81-a22c3e358560 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw namespace: kserve-ci-e2e-test resourceVersion: "13272" uid: b9912495-2ac2-4d13-9bb0-3bcced8805ba kind: Event lastTimestamp: "2026-04-22T14:24:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:54Z" name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw.18a8b3ef7d734110 namespace: kserve-ci-e2e-test resourceVersion: "13548" uid: a4247cbb-2d22-4654-a20a-c452d2c3f65a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw namespace: kserve-ci-e2e-test resourceVersion: "13272" uid: b9912495-2ac2-4d13-9bb0-3bcced8805ba kind: Event lastTimestamp: "2026-04-22T14:24:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:54Z" name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw.18a8b3ef7df20c31 namespace: kserve-ci-e2e-test resourceVersion: "13549" uid: 76d864bd-8235-438a-b84f-894fccd1a63a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw namespace: kserve-ci-e2e-test resourceVersion: "13272" uid: b9912495-2ac2-4d13-9bb0-3bcced8805ba kind: Event lastTimestamp: "2026-04-22T14:25:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:35Z" name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw.18a8b3ef87246d7f namespace: kserve-ci-e2e-test resourceVersion: "13934" uid: 09a46b3f-a8ae-43b3-85c9-3d761a35d1aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw namespace: kserve-ci-e2e-test resourceVersion: "13272" uid: b9912495-2ac2-4d13-9bb0-3bcced8805ba kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-1cfdf-predictor-6985558666-ml6vw.18a8b4010ab79b59 namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: 004725ed-6091-4f59-ade4-70de4e1c7264 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1cfdf-predictor-6985558666 namespace: kserve-ci-e2e-test resourceVersion: "13260" uid: 2a4a0a9f-914e-4152-a8b9-b578896e3b21 kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: 'Created pod: success-200-isvc-1cfdf-predictor-6985558666-ml6vw' metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: success-200-isvc-1cfdf-predictor-6985558666.18a8b3ebfe45ca17 namespace: kserve-ci-e2e-test resourceVersion: "13271" uid: dd4e7e8e-ca88-4f81-a4e1-d1f7419e3772 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13258" uid: 4ed10bfa-add0-4abf-95bd-1829c7a848a4 kind: Event lastTimestamp: "2026-04-22T14:24:39Z" message: Scaled up replica set success-200-isvc-1cfdf-predictor-6985558666 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:39Z" name: success-200-isvc-1cfdf-predictor.18a8b3ebf8b097b7 namespace: kserve-ci-e2e-test resourceVersion: "13263" uid: f2251cd3-774a-40cc-b8ba-2f5a3856eedf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: 3870e0a9-26aa-4419-a7c4-7785e6cf04ff kind: Event lastTimestamp: "2026-04-22T14:25:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:09Z" name: success-200-isvc-1cfdf-predictor.18a8b3ef7f390bd7 namespace: kserve-ci-e2e-test resourceVersion: "13716" uid: 2424d841-9206-42e3-a52a-8a9b1ba84649 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13270" uid: 3870e0a9-26aa-4419-a7c4-7785e6cf04ff kind: Event lastTimestamp: "2026-04-22T14:25:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:09Z" name: success-200-isvc-1cfdf-predictor.18a8b3ef7fa25053 namespace: kserve-ci-e2e-test resourceVersion: "13717" uid: b88183ef-da0f-4505-9a20-52bcf8822253 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:25:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13551" uid: 3870e0a9-26aa-4419-a7c4-7785e6cf04ff kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-1cfdf-predictor.18a8b3f67dd39f23 namespace: kserve-ci-e2e-test resourceVersion: "14269" uid: 28a07c49-5c4e-43f4-a621-74c2adc9ba41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:25:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1cfdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "13551" uid: 3870e0a9-26aa-4419-a7c4-7785e6cf04ff kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-1cfdf-predictor.18a8b3f67e5cced1 namespace: kserve-ci-e2e-test resourceVersion: "14270" uid: 0c351c7a-a1c6-4b9b-a00f-56b73cb73674 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1cfdf namespace: kserve-ci-e2e-test resourceVersion: "13249" uid: 7bef60f7-2340-4562-8861-8e7dbf83dcbc kind: Event lastTimestamp: "2026-04-22T14:25:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:25:45Z" name: success-200-isvc-1cfdf.18a8b3ebfe964131 namespace: kserve-ci-e2e-test resourceVersion: "14039" uid: 8070f252-5d49-4646-8ea4-571c59dcca43 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1cfdf namespace: kserve-ci-e2e-test resourceVersion: "14037" uid: 7bef60f7-2340-4562-8861-8e7dbf83dcbc kind: Event lastTimestamp: "2026-04-22T14:25:45Z" message: InferenceService [success-200-isvc-1cfdf] is Ready metadata: creationTimestamp: "2026-04-22T14:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:25:45Z" name: success-200-isvc-1cfdf.18a8b3fb6b3d29b2 namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: f21f3777-0625-459f-859c-0dae0a93e1a8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:36:14.426563Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9 namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: 980ba7c7-8cc3-4f43-8d02-7158fb43bfe3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-33e8e-predictor-595856cb86-2rfh9 to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9.18a8b48ddf4018f7 namespace: kserve-ci-e2e-test resourceVersion: "19204" uid: 22b338c5-b7fd-4925-9626-c087c8f36de3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9 namespace: kserve-ci-e2e-test resourceVersion: "19209" uid: 980ba7c7-8cc3-4f43-8d02-7158fb43bfe3 kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9.18a8b48de818c0c1 namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: 196a1339-6db5-4290-b7e2-c2b1cb2ca792 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9 namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 980ba7c7-8cc3-4f43-8d02-7158fb43bfe3 kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9.18a8b48de99f77b4 namespace: kserve-ci-e2e-test resourceVersion: "19240" uid: 5d0bf877-b12e-4941-9e07-9209ed52ed36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9 namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 980ba7c7-8cc3-4f43-8d02-7158fb43bfe3 kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9.18a8b48ded1cca01 namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: 185a2a9f-7299-4a65-9288-826dd5604afa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9 namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 980ba7c7-8cc3-4f43-8d02-7158fb43bfe3 kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9.18a8b48ded9f7714 namespace: kserve-ci-e2e-test resourceVersion: "19249" uid: bf7a9600-d7ae-4c84-b8a4-92c25ca02679 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T14:36:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9 namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 980ba7c7-8cc3-4f43-8d02-7158fb43bfe3 kind: Event lastTimestamp: "2026-04-22T14:37:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:36:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:36Z" name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9.18a8b48e26a1d1c7 namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 4f4fbeea-d00d-4ba7-a067-576400449b77 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9 namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 980ba7c7-8cc3-4f43-8d02-7158fb43bfe3 kind: Event lastTimestamp: "2026-04-22T14:37:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:34Z" name: success-200-isvc-33e8e-predictor-595856cb86-2rfh9.18a8b4a08c0ef871 namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: 450a7923-78b8-4671-90ed-282ab4da3100 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-33e8e-predictor-595856cb86 namespace: kserve-ci-e2e-test resourceVersion: "19185" uid: 0f31cd80-b6ec-453e-b9aa-da94e1d2aeeb kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: 'Created pod: success-200-isvc-33e8e-predictor-595856cb86-2rfh9' metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-33e8e-predictor-595856cb86.18a8b48ddebf5a34 namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 2fcecfc0-bd78-4d97-ad4f-884c641195e7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19181" uid: 828f786b-eced-4ded-bcf8-053e67d19d33 kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: Scaled up replica set success-200-isvc-33e8e-predictor-595856cb86 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-33e8e-predictor.18a8b48ddcb019f8 namespace: kserve-ci-e2e-test resourceVersion: "19191" uid: 1cac441d-68bc-4267-b7f4-9f2c33e14d27 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:36:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19184" uid: d0a6da92-b059-4e64-92b6-11e0db15cdfe kind: Event lastTimestamp: "2026-04-22T14:36:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:59Z" name: success-200-isvc-33e8e-predictor.18a8b4915bb7c32f namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 2ef8307a-58e9-4593-8aaa-744fabefa297 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:36:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19184" uid: d0a6da92-b059-4e64-92b6-11e0db15cdfe kind: Event lastTimestamp: "2026-04-22T14:36:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:59Z" name: success-200-isvc-33e8e-predictor.18a8b4915c1d1bc1 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 83fb4caa-7ff1-4a8a-8933-ddc79993034a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:37:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: d0a6da92-b059-4e64-92b6-11e0db15cdfe kind: Event lastTimestamp: "2026-04-22T14:37:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:29Z" name: success-200-isvc-33e8e-predictor.18a8b49bd87ec349 namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 84f36d0f-d3fe-4817-bb33-aba75e3f98b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:37:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-33e8e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: d0a6da92-b059-4e64-92b6-11e0db15cdfe kind: Event lastTimestamp: "2026-04-22T14:37:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:29Z" name: success-200-isvc-33e8e-predictor.18a8b49bd8e9aed5 namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: 636ff9af-c60d-4014-9c4e-faa69890c0c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-33e8e namespace: kserve-ci-e2e-test resourceVersion: "19161" uid: 23c43119-7128-4d57-a6a9-b593b1eb2d3c kind: Event lastTimestamp: "2026-04-22T14:37:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:06Z" name: success-200-isvc-33e8e.18a8b48ddcbc7e47 namespace: kserve-ci-e2e-test resourceVersion: "19894" uid: 7e1eec4c-e274-4136-8c0c-e0768aeb3c5b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-33e8e namespace: kserve-ci-e2e-test resourceVersion: "19161" uid: 23c43119-7128-4d57-a6a9-b593b1eb2d3c kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: 'Failed to update status for InferenceService "success-200-isvc-33e8e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-33e8e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-33e8e.18a8b48ddd414c36 namespace: kserve-ci-e2e-test resourceVersion: "19195" uid: e1c9e72a-e012-4bc1-868e-d731ec6e7db3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-33e8e namespace: kserve-ci-e2e-test resourceVersion: "19161" uid: 23c43119-7128-4d57-a6a9-b593b1eb2d3c kind: Event lastTimestamp: "2026-04-22T14:36:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-33e8e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:14Z" name: success-200-isvc-33e8e.18a8b48ddd41a3a9 namespace: kserve-ci-e2e-test resourceVersion: "19197" uid: bc90a51d-530a-4e4e-a277-f11f4d24c707 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-33e8e namespace: kserve-ci-e2e-test resourceVersion: "19892" uid: 23c43119-7128-4d57-a6a9-b593b1eb2d3c kind: Event lastTimestamp: "2026-04-22T14:37:06Z" message: InferenceService [success-200-isvc-33e8e] is Ready metadata: creationTimestamp: "2026-04-22T14:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:06Z" name: success-200-isvc-33e8e.18a8b49a0a324e7e namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 4015ed03-e71d-44b5-b001-4fa44a96c62c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:36:54.843450Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: a174bb8d-ae21-4fd2-80c9-7369a13331e6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4547e-predictor-776ddf9848-mm7qm to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:36:54Z" name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm.18a8b4974848dff5 namespace: kserve-ci-e2e-test resourceVersion: "19699" uid: d6b4983f-793b-4cd1-a2b8-d8da94146402 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: a174bb8d-ae21-4fd2-80c9-7369a13331e6 kind: Event lastTimestamp: "2026-04-22T14:36:55Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:36:55Z" name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm.18a8b49752752dee namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 07a66997-d3ff-4a2d-a1b3-8d210aa389e8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: a174bb8d-ae21-4fd2-80c9-7369a13331e6 kind: Event lastTimestamp: "2026-04-22T14:36:55Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:55Z" name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm.18a8b49753cfe639 namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e99fb25d-6128-4031-b476-a44e20b30274 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: a174bb8d-ae21-4fd2-80c9-7369a13331e6 kind: Event lastTimestamp: "2026-04-22T14:36:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:55Z" name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm.18a8b4975a95fe3e namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: 49b44fcc-2ce0-4c92-9b6b-cb0dc2041077 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: a174bb8d-ae21-4fd2-80c9-7369a13331e6 kind: Event lastTimestamp: "2026-04-22T14:36:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:55Z" name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm.18a8b4975b1666a4 namespace: kserve-ci-e2e-test resourceVersion: "19758" uid: 6f9d249d-d56f-436a-8970-f4c3b4240e34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:36:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: a174bb8d-ae21-4fd2-80c9-7369a13331e6 kind: Event lastTimestamp: "2026-04-22T14:37:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:36Z" name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm.18a8b4977ee0a865 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 1a794ec7-ede1-484c-bc08-a480d39f26cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: a174bb8d-ae21-4fd2-80c9-7369a13331e6 kind: Event lastTimestamp: "2026-04-22T14:46:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:19Z" name: success-200-isvc-4547e-predictor-776ddf9848-mm7qm.18a8b51adb363037 namespace: kserve-ci-e2e-test resourceVersion: "24117" uid: bbae6800-242e-41a0-b371-94a34a9fe490 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4547e-predictor-776ddf9848 namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: e24ab08b-41c2-43e8-8ddc-e9460283985e kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: 'Created pod: success-200-isvc-4547e-predictor-776ddf9848-mm7qm' metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:54Z" name: success-200-isvc-4547e-predictor-776ddf9848.18a8b49747e20ed3 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: 252876dd-0d4a-415c-a1d0-cd129f3cb28e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: 0eb395d4-b99e-4790-a8fa-546c9b029a26 kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: Scaled up replica set success-200-isvc-4547e-predictor-776ddf9848 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:54Z" name: success-200-isvc-4547e-predictor.18a8b49745f4266b namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: a1663040-8ef9-484f-94a2-d9254fe709bd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:37:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: 63987d30-bcae-4be9-9a14-8b63062c8d02 kind: Event lastTimestamp: "2026-04-22T14:37:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:24Z" name: success-200-isvc-4547e-predictor.18a8b49ac48208c7 namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: 1dd8506a-a06f-4f24-94dc-7e88fcf9ed28 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:37:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: 63987d30-bcae-4be9-9a14-8b63062c8d02 kind: Event lastTimestamp: "2026-04-22T14:37:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:24Z" name: success-200-isvc-4547e-predictor.18a8b49ac4c64ba7 namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: e6f9ed82-9306-47f6-93d3-e35d5799a812 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:37:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 63987d30-bcae-4be9-9a14-8b63062c8d02 kind: Event lastTimestamp: "2026-04-22T14:38:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:09Z" name: success-200-isvc-4547e-predictor.18a8b4a1c2a8a29a namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 45915cd1-c8fe-4910-a09f-abc7a9d44b4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:37:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4547e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 63987d30-bcae-4be9-9a14-8b63062c8d02 kind: Event lastTimestamp: "2026-04-22T14:38:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:09Z" name: success-200-isvc-4547e-predictor.18a8b4a1c30b2c72 namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: c875d5ce-6efd-4377-b2e5-79ef27ebd41a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4547e namespace: kserve-ci-e2e-test resourceVersion: "19663" uid: ab1c1ea6-408a-4f28-a59c-dfa429fa16be kind: Event lastTimestamp: "2026-04-22T14:37:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:46Z" name: success-200-isvc-4547e.18a8b49745cbb47a namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: 08d33a1c-a874-469a-94f7-e5e8e5292afe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4547e namespace: kserve-ci-e2e-test resourceVersion: "19663" uid: ab1c1ea6-408a-4f28-a59c-dfa429fa16be kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: 'Failed to update status for InferenceService "success-200-isvc-4547e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4547e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:54Z" name: success-200-isvc-4547e.18a8b497466b4939 namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: 7d1638bc-352e-4251-b675-c773c3f41d79 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4547e namespace: kserve-ci-e2e-test resourceVersion: "19663" uid: ab1c1ea6-408a-4f28-a59c-dfa429fa16be kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4547e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:54Z" name: success-200-isvc-4547e.18a8b497466b9744 namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 8b6ec3ce-0050-4826-8b37-d31868c471d1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4547e namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: ab1c1ea6-408a-4f28-a59c-dfa429fa16be kind: Event lastTimestamp: "2026-04-22T14:37:46Z" message: InferenceService [success-200-isvc-4547e] is Ready metadata: creationTimestamp: "2026-04-22T14:37:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:46Z" name: success-200-isvc-4547e.18a8b4a362f3c63d namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 0d70cdf4-5263-4c93-8e80-650d3ee51a46 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:46:59.749671Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz namespace: kserve-ci-e2e-test resourceVersion: "24545" uid: 4772875d-82e0-4ea5-8a80-cc629fcd4679 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4e291-predictor-794cfddb48-kvpsz to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:46:59Z" name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz.18a8b5241f8140d2 namespace: kserve-ci-e2e-test resourceVersion: "24553" uid: c2029173-26cd-4cd3-aa6e-d949dba879bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz namespace: kserve-ci-e2e-test resourceVersion: "24566" uid: 4772875d-82e0-4ea5-8a80-cc629fcd4679 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:46:59Z" name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz.18a8b52428d93f2c namespace: kserve-ci-e2e-test resourceVersion: "24594" uid: 203e01ce-9abe-4a90-af4a-b3df44e02079 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz namespace: kserve-ci-e2e-test resourceVersion: "24548" uid: 4772875d-82e0-4ea5-8a80-cc629fcd4679 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:59Z" name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz.18a8b5242a6a65ea namespace: kserve-ci-e2e-test resourceVersion: "24597" uid: 371c657f-c5b5-4484-b3ff-c7db6ab3fa94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz namespace: kserve-ci-e2e-test resourceVersion: "24548" uid: 4772875d-82e0-4ea5-8a80-cc629fcd4679 kind: Event lastTimestamp: "2026-04-22T14:47:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:00Z" name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz.18a8b5243162d613 namespace: kserve-ci-e2e-test resourceVersion: "24607" uid: bf810c73-4f38-4ac8-a92a-3d7ac3b44c65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz namespace: kserve-ci-e2e-test resourceVersion: "24548" uid: 4772875d-82e0-4ea5-8a80-cc629fcd4679 kind: Event lastTimestamp: "2026-04-22T14:47:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:00Z" name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz.18a8b52431e250f1 namespace: kserve-ci-e2e-test resourceVersion: "24608" uid: 73c3f4c9-f8bd-4436-abc9-227c2958232c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T14:47:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz namespace: kserve-ci-e2e-test resourceVersion: "24548" uid: 4772875d-82e0-4ea5-8a80-cc629fcd4679 kind: Event lastTimestamp: "2026-04-22T15:04:31Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T15:04:31Z" name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz.18a8b5245b59a501 namespace: kserve-ci-e2e-test resourceVersion: "32697" uid: 3fa3c980-f403-485c-bbaa-ea652dd58689 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T15:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz namespace: kserve-ci-e2e-test resourceVersion: "24548" uid: 4772875d-82e0-4ea5-8a80-cc629fcd4679 kind: Event lastTimestamp: "2026-04-22T15:04:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T15:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T15:04:29Z" name: success-200-isvc-4e291-predictor-794cfddb48-kvpsz.18a8b6187f2fee8f namespace: kserve-ci-e2e-test resourceVersion: "32635" uid: e370aeb2-939a-4a06-96d5-40831f09e2c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4e291-predictor-794cfddb48 namespace: kserve-ci-e2e-test resourceVersion: "24535" uid: 3d2c3f7e-c064-4a94-9ef3-c0105b55e4c0 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: 'Created pod: success-200-isvc-4e291-predictor-794cfddb48-kvpsz' metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:59Z" name: success-200-isvc-4e291-predictor-794cfddb48.18a8b5241ef03c5e namespace: kserve-ci-e2e-test resourceVersion: "24547" uid: f9043ede-6ac0-4a8b-99dc-9030e902c224 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24526" uid: 20ca013a-ed80-40f0-a623-10219187f553 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: Scaled up replica set success-200-isvc-4e291-predictor-794cfddb48 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:59Z" name: success-200-isvc-4e291-predictor.18a8b5241c571b69 namespace: kserve-ci-e2e-test resourceVersion: "24537" uid: 36f7478b-854a-41d2-9a4a-ef5c5cffab21 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24529" uid: 67202154-71aa-47f8-b864-b833150f833c kind: Event lastTimestamp: "2026-04-22T14:47:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:47:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:14Z" name: success-200-isvc-4e291-predictor.18a8b5279a58befa namespace: kserve-ci-e2e-test resourceVersion: "24761" uid: d6d289d6-f03f-4544-93a3-53ea0ce8b249 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24529" uid: 67202154-71aa-47f8-b864-b833150f833c kind: Event lastTimestamp: "2026-04-22T14:47:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:47:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:14Z" name: success-200-isvc-4e291-predictor.18a8b5279ab3a122 namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: efed8cd9-fb1f-4d0a-8c4b-b538246688c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:47:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24762" uid: 67202154-71aa-47f8-b864-b833150f833c kind: Event lastTimestamp: "2026-04-22T14:48:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:29Z" name: success-200-isvc-4e291-predictor.18a8b52b19e41871 namespace: kserve-ci-e2e-test resourceVersion: "25593" uid: 2c2c0433-e55c-4ada-b2b8-b66eaa706055 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T14:47:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "24762" uid: 67202154-71aa-47f8-b864-b833150f833c kind: Event lastTimestamp: "2026-04-22T14:48:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:29Z" name: success-200-isvc-4e291-predictor.18a8b52b1a9ee18e namespace: kserve-ci-e2e-test resourceVersion: "25594" uid: 2724a27e-be63-45ed-a63e-8d513f9acef8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4e291 namespace: kserve-ci-e2e-test resourceVersion: "24516" uid: 5f60b6db-58fd-42b0-bad8-30dddf24d395 kind: Event lastTimestamp: "2026-04-22T14:47:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:51Z" name: success-200-isvc-4e291.18a8b5241bb27f7c namespace: kserve-ci-e2e-test resourceVersion: "25248" uid: 211b72e9-3368-4f82-a4f3-d0c0ff482503 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4e291 namespace: kserve-ci-e2e-test resourceVersion: "24516" uid: 5f60b6db-58fd-42b0-bad8-30dddf24d395 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: 'Failed to update status for InferenceService "success-200-isvc-4e291": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4e291": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:46:59Z" name: success-200-isvc-4e291.18a8b5241ca1b3de namespace: kserve-ci-e2e-test resourceVersion: "24539" uid: 22f8bbb1-610c-4cf6-b3ee-ab05816121ca reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4e291 namespace: kserve-ci-e2e-test resourceVersion: "24516" uid: 5f60b6db-58fd-42b0-bad8-30dddf24d395 kind: Event lastTimestamp: "2026-04-22T14:46:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4e291": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:46:59Z" name: success-200-isvc-4e291.18a8b5241ca1f2cf namespace: kserve-ci-e2e-test resourceVersion: "24541" uid: 65bbcf04-47f4-4742-9979-320b7183dc61 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4e291 namespace: kserve-ci-e2e-test resourceVersion: "25246" uid: 5f60b6db-58fd-42b0-bad8-30dddf24d395 kind: Event lastTimestamp: "2026-04-22T14:47:51Z" message: InferenceService [success-200-isvc-4e291] is Ready metadata: creationTimestamp: "2026-04-22T14:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:51Z" name: success-200-isvc-4e291.18a8b5303ee0993b namespace: kserve-ci-e2e-test resourceVersion: "25247" uid: 462aea03-ac04-417a-9cae-568f0d4ffbd0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:26:09.654913Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-58a83-predictor-85b474db6d-lpp75 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 0149eb89-a3dc-4a58-8e88-3460aef15834 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-58a83-predictor-85b474db6d-lpp75 to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-58a83-predictor-85b474db6d-lpp75.18a8b401100d0cb1 namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: d623b722-da7a-4d5b-a38a-460e90c20d3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-58a83-predictor-85b474db6d-lpp75 namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 0149eb89-a3dc-4a58-8e88-3460aef15834 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-58a83-predictor-85b474db6d-lpp75.18a8b40119ab7844 namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: ba364b8c-131f-45f3-b6ec-a7c575be3cd7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-58a83-predictor-85b474db6d-lpp75 namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 0149eb89-a3dc-4a58-8e88-3460aef15834 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-58a83-predictor-85b474db6d-lpp75.18a8b4011bf5d233 namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: b6e5d7c5-4cc6-41ae-aa8d-e3c14bddb760 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-58a83-predictor-85b474db6d-lpp75 namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 0149eb89-a3dc-4a58-8e88-3460aef15834 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-58a83-predictor-85b474db6d-lpp75.18a8b40121d37392 namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: 40f4de3c-7bc5-4f87-b96d-97dc589b83c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-58a83-predictor-85b474db6d-lpp75 namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 0149eb89-a3dc-4a58-8e88-3460aef15834 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-58a83-predictor-85b474db6d-lpp75.18a8b401225847c6 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: c7bbd5b6-5626-4640-a1e1-b6c4cce207d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-58a83-predictor-85b474db6d-lpp75 namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 0149eb89-a3dc-4a58-8e88-3460aef15834 kind: Event lastTimestamp: "2026-04-22T14:26:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:51Z" name: success-200-isvc-58a83-predictor-85b474db6d-lpp75.18a8b40149713c92 namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 43a69b15-3cef-48c0-9127-79c07b1f74a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-58a83-predictor-85b474db6d-lpp75 namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 0149eb89-a3dc-4a58-8e88-3460aef15834 kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:34Z" name: success-200-isvc-58a83-predictor-85b474db6d-lpp75.18a8b48492c01262 namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 7616c20c-e7b0-4160-925c-aa06c06e1823 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-58a83-predictor-85b474db6d namespace: kserve-ci-e2e-test resourceVersion: "14328" uid: d3ad0415-74bb-425c-b9bc-cc7d89de7a20 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: 'Created pod: success-200-isvc-58a83-predictor-85b474db6d-lpp75' metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-58a83-predictor-85b474db6d.18a8b4010fa338d8 namespace: kserve-ci-e2e-test resourceVersion: "14341" uid: 7837eaca-a2cf-45d7-b8cb-f60000a5a503 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: a596e75b-adfd-40eb-8050-69d40cc9ca90 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Scaled up replica set success-200-isvc-58a83-predictor-85b474db6d from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-58a83-predictor.18a8b4010e169056 namespace: kserve-ci-e2e-test resourceVersion: "14332" uid: 54050b23-6cd5-4559-9d95-f81b0e832c2b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: 5e46fef5-9ae8-4161-be83-bb0b01e9c338 kind: Event lastTimestamp: "2026-04-22T14:26:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:24Z" name: success-200-isvc-58a83-predictor.18a8b4048c9a0aed namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: b3edbe96-e7e1-489e-8caa-5a745fca531d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: 5e46fef5-9ae8-4161-be83-bb0b01e9c338 kind: Event lastTimestamp: "2026-04-22T14:26:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:24Z" name: success-200-isvc-58a83-predictor.18a8b4048d01ab48 namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: 0c4b7853-c5b7-4eea-8707-438cee5f32bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:26:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 5e46fef5-9ae8-4161-be83-bb0b01e9c338 kind: Event lastTimestamp: "2026-04-22T14:27:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:24Z" name: success-200-isvc-58a83-predictor.18a8b4080c149ab4 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 1a436354-69a6-47c6-874b-4c6966bd4448 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:26:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-58a83-predictor namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 5e46fef5-9ae8-4161-be83-bb0b01e9c338 kind: Event lastTimestamp: "2026-04-22T14:27:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:24Z" name: success-200-isvc-58a83-predictor.18a8b4080c7c4114 namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: 5f234a1c-eb53-4870-b7b1-05474fb02621 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-58a83 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: ba5db619-b1df-4645-b582-cfd453888e39 kind: Event lastTimestamp: "2026-04-22T14:27:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:01Z" name: success-200-isvc-58a83.18a8b4010dd28de3 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 00a6da5d-8113-4185-a8fc-335ffce5da7a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-58a83 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: ba5db619-b1df-4645-b582-cfd453888e39 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: 'Failed to update status for InferenceService "success-200-isvc-58a83": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-58a83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-58a83.18a8b4010e78045c namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 81e652b5-f060-4411-b6c8-83ecdf8fee04 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-58a83 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: ba5db619-b1df-4645-b582-cfd453888e39 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-58a83": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:09Z" name: success-200-isvc-58a83.18a8b4010e78420b namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: aaff5536-d29b-4e31-96a6-665cb64eb703 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-58a83 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: ba5db619-b1df-4645-b582-cfd453888e39 kind: Event lastTimestamp: "2026-04-22T14:27:01Z" message: InferenceService [success-200-isvc-58a83] is Ready metadata: creationTimestamp: "2026-04-22T14:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:01Z" name: success-200-isvc-58a83.18a8b40d2f6c0a15 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 5ca3a025-ba0c-40f7-9246-e255ac2527e3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:35:34.638869Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm namespace: kserve-ci-e2e-test resourceVersion: "18774" uid: 5ea302b2-3e21-4ca0-b1df-eab23c178bde kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:35:34Z" name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm.18a8b4849bb80fe7 namespace: kserve-ci-e2e-test resourceVersion: "18784" uid: ed19f517-7e90-4842-bc7a-b5d2ced0483a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm namespace: kserve-ci-e2e-test resourceVersion: "18789" uid: 5ea302b2-3e21-4ca0-b1df-eab23c178bde kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:35:34Z" name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm.18a8b484a51bf05c namespace: kserve-ci-e2e-test resourceVersion: "18812" uid: 9ceab08b-103c-4b3e-b45b-6be591bf95b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 5ea302b2-3e21-4ca0-b1df-eab23c178bde kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:34Z" name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm.18a8b484a68d05c0 namespace: kserve-ci-e2e-test resourceVersion: "18815" uid: ee1bc652-bcc4-47d2-b6c9-dce0fec1a92c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 5ea302b2-3e21-4ca0-b1df-eab23c178bde kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:34Z" name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm.18a8b484ae0b4a7d namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: 9d268dcc-b1b9-4075-b600-9276c039770d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 5ea302b2-3e21-4ca0-b1df-eab23c178bde kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:34Z" name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm.18a8b484ae82c82d namespace: kserve-ci-e2e-test resourceVersion: "18826" uid: 280c341f-6522-49e6-a3d5-2731ecf8eefa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T14:35:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 5ea302b2-3e21-4ca0-b1df-eab23c178bde kind: Event lastTimestamp: "2026-04-22T14:36:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:56Z" name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm.18a8b484cebc3f2f namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 9ce6a9d9-21c0-4f99-a186-06794b77e2f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 5ea302b2-3e21-4ca0-b1df-eab23c178bde kind: Event lastTimestamp: "2026-04-22T14:36:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:54Z" name: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm.18a8b49747c3687a namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 276851d4-4ea4-4456-9fda-e11009c3abf6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dee6f-predictor-7d88946f7d namespace: kserve-ci-e2e-test resourceVersion: "18754" uid: 516d3d2d-91be-4ff7-be4f-0d99d2df5aba kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: 'Created pod: success-200-isvc-dee6f-predictor-7d88946f7d-b9mxm' metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:34Z" name: success-200-isvc-dee6f-predictor-7d88946f7d.18a8b4849af0c68e namespace: kserve-ci-e2e-test resourceVersion: "18780" uid: b8df3d97-0843-47ad-b380-b0cee16888a6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18752" uid: 3e35fd8c-1a7b-4777-ac30-068ad318bed9 kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Scaled up replica set success-200-isvc-dee6f-predictor-7d88946f7d from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:34Z" name: success-200-isvc-dee6f-predictor.18a8b4849517332f namespace: kserve-ci-e2e-test resourceVersion: "18757" uid: b82083a5-94af-4d06-9965-6c6bb535deea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: 56d4524b-9f0b-4357-a1c9-94d508a3c596 kind: Event lastTimestamp: "2026-04-22T14:36:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:35:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:04Z" name: success-200-isvc-dee6f-predictor.18a8b4881391d176 namespace: kserve-ci-e2e-test resourceVersion: "19063" uid: 830b033f-bdee-4ebe-9969-94cba011e512 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: 56d4524b-9f0b-4357-a1c9-94d508a3c596 kind: Event lastTimestamp: "2026-04-22T14:36:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:35:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:04Z" name: success-200-isvc-dee6f-predictor.18a8b48813d1ada2 namespace: kserve-ci-e2e-test resourceVersion: "19064" uid: 26bf989e-4a81-41db-b72c-87026b595505 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18954" uid: 56d4524b-9f0b-4357-a1c9-94d508a3c596 kind: Event lastTimestamp: "2026-04-22T14:36:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:34Z" name: success-200-isvc-dee6f-predictor.18a8b48f113f47c8 namespace: kserve-ci-e2e-test resourceVersion: "19447" uid: 9a69d7c9-2553-43f0-8a0a-7ac48b03fb62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dee6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18954" uid: 56d4524b-9f0b-4357-a1c9-94d508a3c596 kind: Event lastTimestamp: "2026-04-22T14:36:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:34Z" name: success-200-isvc-dee6f-predictor.18a8b48f119faed5 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 1a4d5f24-2541-4eb9-9097-54b984a740cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dee6f namespace: kserve-ci-e2e-test resourceVersion: "18743" uid: cabf8db3-2089-4267-8e86-fa474ebe1909 kind: Event lastTimestamp: "2026-04-22T14:36:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:26Z" name: success-200-isvc-dee6f.18a8b48494de6389 namespace: kserve-ci-e2e-test resourceVersion: "19378" uid: e931833e-d11e-47dd-8702-09a71ccfa073 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dee6f namespace: kserve-ci-e2e-test resourceVersion: "19376" uid: cabf8db3-2089-4267-8e86-fa474ebe1909 kind: Event lastTimestamp: "2026-04-22T14:36:26Z" message: InferenceService [success-200-isvc-dee6f] is Ready metadata: creationTimestamp: "2026-04-22T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:36:26Z" name: success-200-isvc-dee6f.18a8b490b2516481 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 435273a7-2cca-4dee-bbb0-ab97178fee1f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:46:20.022949Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj namespace: kserve-ci-e2e-test resourceVersion: "24128" uid: 30622015-c7f1-47a6-9b54-e7ed8a2820cd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f4200-predictor-6cc4586c58-bvqcj to ip-10-0-130-255.ec2.internal metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:46:20Z" name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj.18a8b51adf9ba3d2 namespace: kserve-ci-e2e-test resourceVersion: "24142" uid: ca1f0e43-d790-40c3-8e5f-4eaff5540827 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: 30622015-c7f1-47a6-9b54-e7ed8a2820cd kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:46:20Z" name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj.18a8b51ae91e854c namespace: kserve-ci-e2e-test resourceVersion: "24172" uid: e08b5571-722f-4307-8989-3cc11f215c71 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj namespace: kserve-ci-e2e-test resourceVersion: "24138" uid: 30622015-c7f1-47a6-9b54-e7ed8a2820cd kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:20Z" name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj.18a8b51aea968efc namespace: kserve-ci-e2e-test resourceVersion: "24177" uid: 6595e68f-238d-4068-a7bc-536e6d09c4b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj namespace: kserve-ci-e2e-test resourceVersion: "24138" uid: 30622015-c7f1-47a6-9b54-e7ed8a2820cd kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:20Z" name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj.18a8b51af319ff49 namespace: kserve-ci-e2e-test resourceVersion: "24182" uid: 0b84fb49-cbfa-492f-ab35-56e19ddc716e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj namespace: kserve-ci-e2e-test resourceVersion: "24138" uid: 30622015-c7f1-47a6-9b54-e7ed8a2820cd kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:46:20Z" name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj.18a8b51af38ef44d namespace: kserve-ci-e2e-test resourceVersion: "24183" uid: 368997f8-afc9-466a-830d-74b59de6773d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj namespace: kserve-ci-e2e-test resourceVersion: "24138" uid: 30622015-c7f1-47a6-9b54-e7ed8a2820cd kind: Event lastTimestamp: "2026-04-22T14:47:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:41Z" name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj.18a8b51b031bfdc5 namespace: kserve-ci-e2e-test resourceVersion: "25145" uid: c4e05981-38df-4435-b415-e2bbc5976ef3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj namespace: kserve-ci-e2e-test resourceVersion: "24138" uid: 30622015-c7f1-47a6-9b54-e7ed8a2820cd kind: Event lastTimestamp: "2026-04-22T14:47:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:47:40Z" name: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj.18a8b52d95071747 namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 24fa37ce-075d-4535-9ef9-2f8862803bd8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-255.ec2.internal source: component: kubelet host: ip-10-0-130-255.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f4200-predictor-6cc4586c58 namespace: kserve-ci-e2e-test resourceVersion: "24102" uid: 01a5b0d0-695a-4afe-95ac-7cfd6c75bbd5 kind: Event lastTimestamp: "2026-04-22T14:46:20Z" message: 'Created pod: success-200-isvc-f4200-predictor-6cc4586c58-bvqcj' metadata: creationTimestamp: "2026-04-22T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:20Z" name: success-200-isvc-f4200-predictor-6cc4586c58.18a8b51ade9df761 namespace: kserve-ci-e2e-test resourceVersion: "24134" uid: ba17f15b-9bf4-4d66-94fc-b81f02d3483c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:46:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24097" uid: 6c17caea-e0bc-47d6-b65c-7a017ab54b37 kind: Event lastTimestamp: "2026-04-22T14:46:19Z" message: Scaled up replica set success-200-isvc-f4200-predictor-6cc4586c58 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:19Z" name: success-200-isvc-f4200-predictor.18a8b51ad8cd06dd namespace: kserve-ci-e2e-test resourceVersion: "24106" uid: 6518ebac-6793-4c12-872e-aec223efa1a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:46:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24101" uid: 2df0374a-820b-43ad-8dc1-68538616150d kind: Event lastTimestamp: "2026-04-22T14:46:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:49Z" name: success-200-isvc-f4200-predictor.18a8b51e574cfd03 namespace: kserve-ci-e2e-test resourceVersion: "24414" uid: 17834089-bc16-4130-afed-ed20e0f99643 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:46:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24101" uid: 2df0374a-820b-43ad-8dc1-68538616150d kind: Event lastTimestamp: "2026-04-22T14:46:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:46:49Z" name: success-200-isvc-f4200-predictor.18a8b51e578f8baa namespace: kserve-ci-e2e-test resourceVersion: "24415" uid: f58f9855-80fc-4e66-8373-16d3ba2946c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24309" uid: 2df0374a-820b-43ad-8dc1-68538616150d kind: Event lastTimestamp: "2026-04-22T14:47:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:34Z" name: success-200-isvc-f4200-predictor.18a8b525554b3655 namespace: kserve-ci-e2e-test resourceVersion: "24969" uid: 307e86cf-0f32-47ee-80dc-1b2cab50a0d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f4200-predictor namespace: kserve-ci-e2e-test resourceVersion: "24309" uid: 2df0374a-820b-43ad-8dc1-68538616150d kind: Event lastTimestamp: "2026-04-22T14:47:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:47:34Z" name: success-200-isvc-f4200-predictor.18a8b525559e4213 namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: 917ee8bb-add5-403e-9272-06c6c4303d15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:46:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f4200 namespace: kserve-ci-e2e-test resourceVersion: "24088" uid: 059d43a8-6f50-4dc7-900b-7363375bbd31 kind: Event lastTimestamp: "2026-04-22T14:47:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:11Z" name: success-200-isvc-f4200.18a8b51ad85627e7 namespace: kserve-ci-e2e-test resourceVersion: "24741" uid: ec00c9a8-0b08-4ace-876a-ef15bd486863 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:47:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f4200 namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 059d43a8-6f50-4dc7-900b-7363375bbd31 kind: Event lastTimestamp: "2026-04-22T14:47:11Z" message: InferenceService [success-200-isvc-f4200] is Ready metadata: creationTimestamp: "2026-04-22T14:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:47:11Z" name: success-200-isvc-f4200.18a8b526e72f3950 namespace: kserve-ci-e2e-test resourceVersion: "24740" uid: d99f018a-89ea-45b7-8314-50d4c404970d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:25:59.284319Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1cfdf-5887c7bb65-mq9nr namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: 733ef929-3e43-46f7-80b0-1b8a0d2bcd13 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1cfdf-5887c7bb65-mq9nr to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:25:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:25:59Z" name: switch-graph-1cfdf-5887c7bb65-mq9nr.18a8b3fea5ea6ce5 namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: 99111eb2-bc2c-4e09-94ea-8fc910e51d1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:59Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1cfdf-5887c7bb65-mq9nr namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 733ef929-3e43-46f7-80b0-1b8a0d2bcd13 kind: Event lastTimestamp: "2026-04-22T14:25:59Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:25:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:25:59Z" name: switch-graph-1cfdf-5887c7bb65-mq9nr.18a8b3febf914f2c namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: 35603257-40c0-4bab-8f2f-4989db34a4ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1cfdf} kind: Pod name: switch-graph-1cfdf-5887c7bb65-mq9nr namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 733ef929-3e43-46f7-80b0-1b8a0d2bcd13 kind: Event lastTimestamp: "2026-04-22T14:25:59Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" metadata: creationTimestamp: "2026-04-22T14:25:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:59Z" name: switch-graph-1cfdf-5887c7bb65-mq9nr.18a8b3fec0eb90d3 namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 60ef3b89-d058-4749-ba0c-f1f9da9fdd07 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1cfdf} kind: Pod name: switch-graph-1cfdf-5887c7bb65-mq9nr namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 733ef929-3e43-46f7-80b0-1b8a0d2bcd13 kind: Event lastTimestamp: "2026-04-22T14:26:02Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" in 2.276s (2.276s including waiting). Image size: 216233534 bytes.' metadata: creationTimestamp: "2026-04-22T14:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:02Z" name: switch-graph-1cfdf-5887c7bb65-mq9nr.18a8b3ff48988cb9 namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 594a32de-18b2-477d-80d9-52e9761132b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1cfdf} kind: Pod name: switch-graph-1cfdf-5887c7bb65-mq9nr namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 733ef929-3e43-46f7-80b0-1b8a0d2bcd13 kind: Event lastTimestamp: "2026-04-22T14:26:02Z" message: 'Created container: switch-graph-1cfdf' metadata: creationTimestamp: "2026-04-22T14:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:02Z" name: switch-graph-1cfdf-5887c7bb65-mq9nr.18a8b3ff4b821bd2 namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: e64cd5d9-a7a4-401f-9e1d-883c33fa3086 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1cfdf} kind: Pod name: switch-graph-1cfdf-5887c7bb65-mq9nr namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 733ef929-3e43-46f7-80b0-1b8a0d2bcd13 kind: Event lastTimestamp: "2026-04-22T14:26:02Z" message: Started container switch-graph-1cfdf metadata: creationTimestamp: "2026-04-22T14:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:02Z" name: switch-graph-1cfdf-5887c7bb65-mq9nr.18a8b3ff4bec73a1 namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: ebe1a9a9-0943-449a-a714-632415aaf6ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1cfdf} kind: Pod name: switch-graph-1cfdf-5887c7bb65-mq9nr namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 733ef929-3e43-46f7-80b0-1b8a0d2bcd13 kind: Event lastTimestamp: "2026-04-22T14:26:09Z" message: Stopping container switch-graph-1cfdf metadata: creationTimestamp: "2026-04-22T14:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:09Z" name: switch-graph-1cfdf-5887c7bb65-mq9nr.18a8b4010438541c namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: d590a8c5-80a2-466c-acd9-264256e80f97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:26:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1cfdf} kind: Pod name: switch-graph-1cfdf-5887c7bb65-mq9nr namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 733ef929-3e43-46f7-80b0-1b8a0d2bcd13 kind: Event lastTimestamp: "2026-04-22T14:26:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:38Z" name: switch-graph-1cfdf-5887c7bb65-mq9nr.18a8b401fa858907 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: e3a166ab-a0fe-4638-b127-49d63eb62590 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1cfdf-5887c7bb65 namespace: kserve-ci-e2e-test resourceVersion: "14157" uid: 1fd2b364-82cf-4143-a035-7ca0cb8a99e8 kind: Event lastTimestamp: "2026-04-22T14:25:59Z" message: 'Created pod: switch-graph-1cfdf-5887c7bb65-mq9nr' metadata: creationTimestamp: "2026-04-22T14:25:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:59Z" name: switch-graph-1cfdf-5887c7bb65.18a8b3fea57c8789 namespace: kserve-ci-e2e-test resourceVersion: "14162" uid: 4f7b44fb-57db-46d2-b71b-ac171e9e5fb0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1cfdf namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: f9a110c0-3231-4c3a-a562-3aac096df6f2 kind: Event lastTimestamp: "2026-04-22T14:25:59Z" message: Scaled up replica set switch-graph-1cfdf-5887c7bb65 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:25:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:59Z" name: switch-graph-1cfdf.18a8b3fea48284cc namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: 0e7c45e5-f818-4d00-bcf8-3de15d1a2e1d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1cfdf namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: dafa368b-b9f8-4d5a-abd2-adeed5e66957 kind: Event lastTimestamp: "2026-04-22T14:25:59Z" message: InferenceGraph [switch-graph-1cfdf] is Ready metadata: creationTimestamp: "2026-04-22T14:25:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:25:59Z" name: switch-graph-1cfdf.18a8b3fea908236e namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: 45fb7a7c-6a34-49d8-972f-addcf4bc0f7e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:48:09.759335Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4e291-65ccfb5fb-lzpv7 namespace: kserve-ci-e2e-test resourceVersion: "25399" uid: d6062ebe-8887-4cb7-b11c-af631c1f9061 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-4e291-65ccfb5fb-lzpv7 to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:48:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:48:09Z" name: switch-graph-4e291-65ccfb5fb-lzpv7.18a8b5346c6803f0 namespace: kserve-ci-e2e-test resourceVersion: "25404" uid: 42f26b96-4778-47f3-9322-e1ecd0b1d2eb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:09Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4e291-65ccfb5fb-lzpv7 namespace: kserve-ci-e2e-test resourceVersion: "25401" uid: d6062ebe-8887-4cb7-b11c-af631c1f9061 kind: Event lastTimestamp: "2026-04-22T14:48:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-4e291-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T14:48:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:09Z" name: switch-graph-4e291-65ccfb5fb-lzpv7.18a8b5347597cfa6 namespace: kserve-ci-e2e-test resourceVersion: "25414" uid: cd5c856c-dd7f-4780-9ec3-0f70780ed95e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:10Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4e291-65ccfb5fb-lzpv7 namespace: kserve-ci-e2e-test resourceVersion: "25407" uid: d6062ebe-8887-4cb7-b11c-af631c1f9061 kind: Event lastTimestamp: "2026-04-22T14:48:10Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:48:10Z" name: switch-graph-4e291-65ccfb5fb-lzpv7.18a8b534a8d66fc5 namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 6e72dfec-cfb1-4d21-9a17-caaac2721479 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4e291} kind: Pod name: switch-graph-4e291-65ccfb5fb-lzpv7 namespace: kserve-ci-e2e-test resourceVersion: "25401" uid: d6062ebe-8887-4cb7-b11c-af631c1f9061 kind: Event lastTimestamp: "2026-04-22T14:48:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:10Z" name: switch-graph-4e291-65ccfb5fb-lzpv7.18a8b534aae4e75e namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: 8fa5c4a7-9107-42df-8d8b-b49f6cb59a32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4e291} kind: Pod name: switch-graph-4e291-65ccfb5fb-lzpv7 namespace: kserve-ci-e2e-test resourceVersion: "25401" uid: d6062ebe-8887-4cb7-b11c-af631c1f9061 kind: Event lastTimestamp: "2026-04-22T14:48:10Z" message: 'Created container: switch-graph-4e291' metadata: creationTimestamp: "2026-04-22T14:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:10Z" name: switch-graph-4e291-65ccfb5fb-lzpv7.18a8b534adfbc159 namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: 0be4d33c-b626-407d-b6f3-4db0ef282b1f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4e291} kind: Pod name: switch-graph-4e291-65ccfb5fb-lzpv7 namespace: kserve-ci-e2e-test resourceVersion: "25401" uid: d6062ebe-8887-4cb7-b11c-af631c1f9061 kind: Event lastTimestamp: "2026-04-22T14:48:10Z" message: Started container switch-graph-4e291 metadata: creationTimestamp: "2026-04-22T14:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:48:10Z" name: switch-graph-4e291-65ccfb5fb-lzpv7.18a8b534ae66f7fd namespace: kserve-ci-e2e-test resourceVersion: "25434" uid: 51e9d213-c979-48f9-8d83-49bec99bf710 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T15:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4e291} kind: Pod name: switch-graph-4e291-65ccfb5fb-lzpv7 namespace: kserve-ci-e2e-test resourceVersion: "25401" uid: d6062ebe-8887-4cb7-b11c-af631c1f9061 kind: Event lastTimestamp: "2026-04-22T15:04:29Z" message: Stopping container switch-graph-4e291 metadata: creationTimestamp: "2026-04-22T15:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T15:04:29Z" name: switch-graph-4e291-65ccfb5fb-lzpv7.18a8b61874042ab9 namespace: kserve-ci-e2e-test resourceVersion: "32621" uid: ae856774-2d50-4c8d-a8be-8f3b5fb133c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T15:04:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4e291} kind: Pod name: switch-graph-4e291-65ccfb5fb-lzpv7 namespace: kserve-ci-e2e-test resourceVersion: "25401" uid: d6062ebe-8887-4cb7-b11c-af631c1f9061 kind: Event lastTimestamp: "2026-04-22T15:04:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T15:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T15:04:52Z" name: switch-graph-4e291-65ccfb5fb-lzpv7.18a8b6193518cc86 namespace: kserve-ci-e2e-test resourceVersion: "32855" uid: c75e9240-156b-4d62-adf3-cf85c84b8870 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-4e291-65ccfb5fb namespace: kserve-ci-e2e-test resourceVersion: "25396" uid: 98799534-0c56-48a6-b4ac-ece592777cf7 kind: Event lastTimestamp: "2026-04-22T14:48:09Z" message: 'Created pod: switch-graph-4e291-65ccfb5fb-lzpv7' metadata: creationTimestamp: "2026-04-22T14:48:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:09Z" name: switch-graph-4e291-65ccfb5fb.18a8b5346bd3b4a6 namespace: kserve-ci-e2e-test resourceVersion: "25400" uid: 4279c0e5-83da-4e6f-8497-17a3c7c51d2c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-4e291 namespace: kserve-ci-e2e-test resourceVersion: "25394" uid: b2bd199f-f832-4a45-98b0-88f27c1792a0 kind: Event lastTimestamp: "2026-04-22T14:48:09Z" message: Scaled up replica set switch-graph-4e291-65ccfb5fb from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:48:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:09Z" name: switch-graph-4e291.18a8b5346aef8e4e namespace: kserve-ci-e2e-test resourceVersion: "25397" uid: 3c16abba-3e70-439f-9387-708801b37ad0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:48:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4e291 namespace: kserve-ci-e2e-test resourceVersion: "25410" uid: e0081d3b-6e0b-4899-8adc-83d7724797a3 kind: Event lastTimestamp: "2026-04-22T14:48:09Z" message: InferenceGraph [switch-graph-4e291] is Ready metadata: creationTimestamp: "2026-04-22T14:48:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:48:09Z" name: switch-graph-4e291.18a8b53470337fb1 namespace: kserve-ci-e2e-test resourceVersion: "25412" uid: cfce502e-c705-4282-8278-75278b98b116 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:48:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4e291 namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: 1ef2089b-ed13-4e90-bf13-2a15181e8e7d kind: Event lastTimestamp: "2026-04-22T14:48:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:48:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:39Z" name: switch-graph-4e291.18a8b537ea82bb2c namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: d89620ca-5085-4e03-9f40-21c8c49876e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:48:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4e291 namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: 1ef2089b-ed13-4e90-bf13-2a15181e8e7d kind: Event lastTimestamp: "2026-04-22T14:48:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:48:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:48:39Z" name: switch-graph-4e291.18a8b537eb1ddba8 namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: 6eeaccb4-0db3-41f4-91f2-27e9998b8781 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:27:19.750404Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-58a83-5fcb457996-rbdg5 namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: 2e233ceb-a8a0-41b3-b26a-781b386f0396 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-58a83-5fcb457996-rbdg5 to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-04-22T14:27:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:27:19Z" name: switch-graph-58a83-5fcb457996-rbdg5.18a8b41162115ce9 namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: ac367bfc-035c-4081-936c-3f091b518832 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-9647f975b-ptn22 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-58a83-5fcb457996-rbdg5 namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 2e233ceb-a8a0-41b3-b26a-781b386f0396 kind: Event lastTimestamp: "2026-04-22T14:27:20Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:27:20Z" name: switch-graph-58a83-5fcb457996-rbdg5.18a8b4117a9f9f82 namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: e42b21f2-dcfb-4ca3-ad29-b64c7da6577d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-58a83} kind: Pod name: switch-graph-58a83-5fcb457996-rbdg5 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 2e233ceb-a8a0-41b3-b26a-781b386f0396 kind: Event lastTimestamp: "2026-04-22T14:27:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:27:20Z" name: switch-graph-58a83-5fcb457996-rbdg5.18a8b4117c9fa105 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: c8962a8b-73a7-4943-ad25-92b11dc10aac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-58a83} kind: Pod name: switch-graph-58a83-5fcb457996-rbdg5 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 2e233ceb-a8a0-41b3-b26a-781b386f0396 kind: Event lastTimestamp: "2026-04-22T14:27:20Z" message: 'Created container: switch-graph-58a83' metadata: creationTimestamp: "2026-04-22T14:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:27:20Z" name: switch-graph-58a83-5fcb457996-rbdg5.18a8b4117f15a413 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 1f7e40f0-d3b5-4c59-8779-a0a4a52c2178 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-58a83} kind: Pod name: switch-graph-58a83-5fcb457996-rbdg5 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 2e233ceb-a8a0-41b3-b26a-781b386f0396 kind: Event lastTimestamp: "2026-04-22T14:27:20Z" message: Started container switch-graph-58a83 metadata: creationTimestamp: "2026-04-22T14:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:27:20Z" name: switch-graph-58a83-5fcb457996-rbdg5.18a8b4117f782626 namespace: kserve-ci-e2e-test resourceVersion: "15238" uid: c7d330f6-f38b-42ab-b553-429647251ed9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-58a83} kind: Pod name: switch-graph-58a83-5fcb457996-rbdg5 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 2e233ceb-a8a0-41b3-b26a-781b386f0396 kind: Event lastTimestamp: "2026-04-22T14:35:34Z" message: Stopping container switch-graph-58a83 metadata: creationTimestamp: "2026-04-22T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:34Z" name: switch-graph-58a83-5fcb457996-rbdg5.18a8b4848ccf8e90 namespace: kserve-ci-e2e-test resourceVersion: "18728" uid: 95159b4d-f7e9-4b94-b4d0-4e8748297a0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-58a83} kind: Pod name: switch-graph-58a83-5fcb457996-rbdg5 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 2e233ceb-a8a0-41b3-b26a-781b386f0396 kind: Event lastTimestamp: "2026-04-22T14:36:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:01Z" name: switch-graph-58a83-5fcb457996-rbdg5.18a8b4851d606534 namespace: kserve-ci-e2e-test resourceVersion: "19042" uid: 4ceb0799-fde1-406a-a594-2f07d132d88a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-58a83-5fcb457996 namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 30b7b159-491e-49c0-a987-d69676b455c2 kind: Event lastTimestamp: "2026-04-22T14:27:19Z" message: 'Created pod: switch-graph-58a83-5fcb457996-rbdg5' metadata: creationTimestamp: "2026-04-22T14:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:19Z" name: switch-graph-58a83-5fcb457996.18a8b4116158de16 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 3362fe67-8f70-4ca6-8020-748da9ec42b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-58a83 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 99327d94-64ce-41ce-b35b-aeedff5a0e1c kind: Event lastTimestamp: "2026-04-22T14:27:19Z" message: Scaled up replica set switch-graph-58a83-5fcb457996 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:19Z" name: switch-graph-58a83.18a8b411607fbe13 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: 3bdad908-fca3-437c-9dc0-4d533718fb8f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-58a83 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 5b23f05a-e46e-43d2-bb22-de54c884a98b kind: Event lastTimestamp: "2026-04-22T14:27:19Z" message: InferenceGraph [switch-graph-58a83] is Ready metadata: creationTimestamp: "2026-04-22T14:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:19Z" name: switch-graph-58a83.18a8b411668e0df0 namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: b606090e-4091-48c5-b3df-bd2bebcadc5b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-58a83 namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: bdb4e83b-a2cb-4f4f-8fd6-c3e3d4c70ba9 kind: Event lastTimestamp: "2026-04-22T14:27:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:49Z" name: switch-graph-58a83.18a8b414df521bfa namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 4fc8636f-1635-4e44-a1d0-2a4758a935ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-58a83 namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: bdb4e83b-a2cb-4f4f-8fd6-c3e3d4c70ba9 kind: Event lastTimestamp: "2026-04-22T14:27:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:49Z" name: switch-graph-58a83.18a8b414dfbe0f72 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 4346e543-49e1-4574-802f-b03b41ab1de8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32869"