--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:06.505194Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1dabb-976bfb698-7kt9p namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: 420f11c7-ea6b-41a3-b4e9-64434becd5ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-1dabb-976bfb698-7kt9p to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:06Z" name: ensemble-graph-1dabb-976bfb698-7kt9p.18a9696412aad967 namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: 6747fe88-872d-488f-b03e-38a5fa2a69a5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:06Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1dabb-976bfb698-7kt9p namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 420f11c7-ea6b-41a3-b4e9-64434becd5ba kind: Event lastTimestamp: "2026-04-24T21:50:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-1dabb-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:06Z" name: ensemble-graph-1dabb-976bfb698-7kt9p.18a969641fa2b074 namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: d56f40a8-5b36-4399-919c-2dff6ac7447d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:07Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1dabb-976bfb698-7kt9p namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 420f11c7-ea6b-41a3-b4e9-64434becd5ba kind: Event lastTimestamp: "2026-04-24T21:50:07Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:07Z" name: ensemble-graph-1dabb-976bfb698-7kt9p.18a969645019ce99 namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: 1301dcff-3e6f-409f-b82c-7a179daed515 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1dabb} kind: Pod name: ensemble-graph-1dabb-976bfb698-7kt9p namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 420f11c7-ea6b-41a3-b4e9-64434becd5ba kind: Event lastTimestamp: "2026-04-24T21:50:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:07Z" name: ensemble-graph-1dabb-976bfb698-7kt9p.18a969645223c2f8 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 323a4633-7610-4d71-94c8-a0732f0e4e1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1dabb} kind: Pod name: ensemble-graph-1dabb-976bfb698-7kt9p namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 420f11c7-ea6b-41a3-b4e9-64434becd5ba kind: Event lastTimestamp: "2026-04-24T21:50:07Z" message: 'Created container: ensemble-graph-1dabb' metadata: creationTimestamp: "2026-04-24T21:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:07Z" name: ensemble-graph-1dabb-976bfb698-7kt9p.18a96964551cab51 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: a2417aa0-a9a0-4e46-b642-6e7d76f96048 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1dabb} kind: Pod name: ensemble-graph-1dabb-976bfb698-7kt9p namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 420f11c7-ea6b-41a3-b4e9-64434becd5ba kind: Event lastTimestamp: "2026-04-24T21:50:07Z" message: Started container ensemble-graph-1dabb metadata: creationTimestamp: "2026-04-24T21:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:07Z" name: ensemble-graph-1dabb-976bfb698-7kt9p.18a96964559539ec namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 07415f91-27ad-4900-89a7-0610d08674c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1dabb} kind: Pod name: ensemble-graph-1dabb-976bfb698-7kt9p namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 420f11c7-ea6b-41a3-b4e9-64434becd5ba kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Stopping container ensemble-graph-1dabb metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: ensemble-graph-1dabb-976bfb698-7kt9p.18a969d73b1be1d5 namespace: kserve-ci-e2e-test resourceVersion: "24682" uid: 33334dc4-6111-4f06-8b37-00c3d84b4da3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1dabb} kind: Pod name: ensemble-graph-1dabb-976bfb698-7kt9p namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 420f11c7-ea6b-41a3-b4e9-64434becd5ba kind: Event lastTimestamp: "2026-04-24T21:58:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:49Z" name: ensemble-graph-1dabb-976bfb698-7kt9p.18a969d80234eaf9 namespace: kserve-ci-e2e-test resourceVersion: "25020" uid: a23b0e52-6841-47db-8582-560a1fbaa91e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-1dabb-976bfb698 namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 9070a0eb-1c9a-45ef-8fd1-18d52447eee6 kind: Event lastTimestamp: "2026-04-24T21:50:06Z" message: 'Created pod: ensemble-graph-1dabb-976bfb698-7kt9p' metadata: creationTimestamp: "2026-04-24T21:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:06Z" name: ensemble-graph-1dabb-976bfb698.18a969641248eb9f namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 1855700d-cd04-482f-b56e-868ba5277f7a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: 0bb1542e-004b-43bc-9d02-6b35265158ae kind: Event lastTimestamp: "2026-04-24T21:50:06Z" message: Scaled up replica set ensemble-graph-1dabb-976bfb698 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:06Z" name: ensemble-graph-1dabb.18a9696411054330 namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: 926da47d-66ea-48a1-b4f9-394fcff328d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 2557a3cf-e870-4b05-b81a-fac32a19871c kind: Event lastTimestamp: "2026-04-24T21:50:06Z" message: InferenceGraph [ensemble-graph-1dabb] is Ready metadata: creationTimestamp: "2026-04-24T21:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:06Z" name: ensemble-graph-1dabb.18a9696416bff691 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: be6b5aa5-c223-45f4-998f-a11a3531c897 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 2557a3cf-e870-4b05-b81a-fac32a19871c kind: Event lastTimestamp: "2026-04-24T21:50:06Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-1dabb": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-1dabb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:06Z" name: ensemble-graph-1dabb.18a969641900158d namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 770492cd-b922-4ee9-b6dd-71e77fc1e513 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 2557a3cf-e870-4b05-b81a-fac32a19871c kind: Event lastTimestamp: "2026-04-24T21:50:06Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-1dabb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:06Z" name: ensemble-graph-1dabb.18a969641900660b namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 365be59f-76aa-46db-b60a-83a0539ec190 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 7c9b31b3-0cde-4828-beb9-a119629172e6 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:21Z" name: ensemble-graph-1dabb.18a969678f8c5601 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: c4f24fea-d0be-414f-9d78-c290df629ee4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 7c9b31b3-0cde-4828-beb9-a119629172e6 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:21Z" name: ensemble-graph-1dabb.18a969678ff64f77 namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: 184947ed-6746-451c-9ae5-7c0173044ccf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: 7c9b31b3-0cde-4828-beb9-a119629172e6 kind: Event lastTimestamp: "2026-04-24T21:50:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:36Z" name: ensemble-graph-1dabb.18a9696b0ea2eb3d namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: 557a7932-205c-438e-b482-37cb7ae98aaf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: 7c9b31b3-0cde-4828-beb9-a119629172e6 kind: Event lastTimestamp: "2026-04-24T21:50:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:36Z" name: ensemble-graph-1dabb.18a9696b0f087459 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: b5dab26d-4745-403d-8ab0-f1baabfb916a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:48:46.194243Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2525b-8997d599-kctgl namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: aadad426-f0be-479b-9a3b-c0b06acec4c4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-2525b-8997d599-kctgl to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b-8997d599-kctgl.18a969515fc2feb1 namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 31a8f41e-ae79-4f28-8353-d1d33a4c77b4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:46Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2525b-8997d599-kctgl namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: aadad426-f0be-479b-9a3b-c0b06acec4c4 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b-8997d599-kctgl.18a96951791266f6 namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: a63586f5-0448-4dea-bdd4-899193b1795a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2525b} kind: Pod name: ensemble-graph-2525b-8997d599-kctgl namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: aadad426-f0be-479b-9a3b-c0b06acec4c4 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b-8997d599-kctgl.18a969517a6aad68 namespace: kserve-ci-e2e-test resourceVersion: "20139" uid: c008f076-61e2-4fea-b0a3-355e6db8ea03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2525b} kind: Pod name: ensemble-graph-2525b-8997d599-kctgl namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: aadad426-f0be-479b-9a3b-c0b06acec4c4 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: 'Created container: ensemble-graph-2525b' metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b-8997d599-kctgl.18a969517d8ff36a namespace: kserve-ci-e2e-test resourceVersion: "20140" uid: 48dcf03a-fd48-4cea-8490-e31b59ce13bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2525b} kind: Pod name: ensemble-graph-2525b-8997d599-kctgl namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: aadad426-f0be-479b-9a3b-c0b06acec4c4 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: Started container ensemble-graph-2525b metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b-8997d599-kctgl.18a969517e06d9a7 namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: 79ebeabe-44b4-49fb-86a1-db0c5874c402 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2525b} kind: Pod name: ensemble-graph-2525b-8997d599-kctgl namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: aadad426-f0be-479b-9a3b-c0b06acec4c4 kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: Stopping container ensemble-graph-2525b metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:56Z" name: ensemble-graph-2525b-8997d599-kctgl.18a96953b8153af7 namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: bfd67420-b36c-47c3-8a4a-063020798e95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:48:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2525b} kind: Pod name: ensemble-graph-2525b-8997d599-kctgl namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: aadad426-f0be-479b-9a3b-c0b06acec4c4 kind: Event lastTimestamp: "2026-04-24T21:49:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:23Z" name: ensemble-graph-2525b-8997d599-kctgl.18a969542b5bb7b2 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 21fda343-b6d7-441f-bdb6-3bfbe4a8279a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-2525b-8997d599 namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 7431324a-fef7-4bb7-922f-cbaaba074cd0 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: 'Created pod: ensemble-graph-2525b-8997d599-kctgl' metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b-8997d599.18a969515f6019cb namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: c0ae7042-70d9-438c-883f-2a78a4be19f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-2525b namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 66147703-3d14-4207-b7b0-8da4a2a48f65 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: Scaled up replica set ensemble-graph-2525b-8997d599 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b.18a969515ec7579d namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: a878a8ee-abdb-4cc8-9785-85c5f133a3d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-2525b namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: 6581a69e-d492-4f9e-99ac-1532f9861736 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: InferenceGraph [ensemble-graph-2525b] is Ready metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b.18a96951649eeb51 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 64cc06d0-fbd3-454e-ba32-4072656d3958 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-2525b namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: 6581a69e-d492-4f9e-99ac-1532f9861736 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-2525b": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-2525b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b.18a969516720bef7 namespace: kserve-ci-e2e-test resourceVersion: "20131" uid: 57391df1-78b1-46af-ab9a-9808a0efe0ea reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-2525b namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: 6581a69e-d492-4f9e-99ac-1532f9861736 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-2525b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:46Z" name: ensemble-graph-2525b.18a9695167210c13 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: b6c31e8a-f981-45e1-bed1-cc81018d4f3b reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:48:56.628760Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1dabb-predictor-64d84476b8-kbgzx to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:48:56Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a96953cdb50f5f namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: a444f9a6-131e-4a0f-8319-2b95cea9ef87 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:48:57Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a96953f3ae4fdc namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: b93cc10f-924b-4187-9109-e9b062f67598 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a96953f5c9c6c3 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 7f5e942e-40f6-413b-bd25-bf8b8e5754d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a96953f8c5eee4 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: fc0777ea-0ff3-4ab7-8ac0-0dfad7e0e373 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a96953f934eecb namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 2dab68bf-1c4f-4f60-bf85-99947314fff3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a96953f940e35b namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 1359f830-c115-4cbd-af48-426d97f38c66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a96953fcb1c1ec namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: c3c6f4c6-89f0-4aa6-be76-2183353d5dc7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a96953fd256a96 namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: 4bba95d3-26d8-4f37-87d6-c61daa87803d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:58:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:23Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a969541b4947ba namespace: kserve-ci-e2e-test resourceVersion: "24822" uid: 4603b383-1c5b-4db7-9d94-1f0344247c11 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a969d7474feb6e namespace: kserve-ci-e2e-test resourceVersion: "24733" uid: aac33c41-504a-47e0-a6cf-70d303b2e8b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a969d74750f174 namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: fdb272a8-9eb6-4ad8-bda1-918ac5eab19e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ea3669b0-a0cd-418f-9778-36c1b7855246 kind: Event lastTimestamp: "2026-04-24T21:58:23Z" message: 'Readiness probe failed: Get "https://10.134.0.47:8643/healthz": dial tcp 10.134.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:23Z" name: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx.18a969d7e3af6c36 namespace: kserve-ci-e2e-test resourceVersion: "24821" uid: 333c9723-dd70-44be-834f-7199d8c29735 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1dabb-predictor-64d84476b8 namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 5b21b97f-8541-4241-b93c-2dc55374a11c kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'Created pod: error-404-isvc-1dabb-predictor-64d84476b8-kbgzx' metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:56Z" name: error-404-isvc-1dabb-predictor-64d84476b8.18a96953cd4de7dd namespace: kserve-ci-e2e-test resourceVersion: "20320" uid: 2ebfcde7-9c76-466c-981e-bfd350c9797f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 2641e149-f69b-4778-bf7c-93b9b1f46a8b kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: Scaled up replica set error-404-isvc-1dabb-predictor-64d84476b8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:56Z" name: error-404-isvc-1dabb-predictor.18a96953cb64c2df namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: 9dd1fea1-fd21-45bf-b46c-ac9c1fdf9133 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: e155c2fe-6d5a-4b4f-bc63-af47d2b53ffe kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: error-404-isvc-1dabb-predictor.18a9695749ee47fb namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 4d8963e4-213e-4f96-bea2-f8ed46cfd85e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: e155c2fe-6d5a-4b4f-bc63-af47d2b53ffe kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: error-404-isvc-1dabb-predictor.18a969574a5b8b70 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 1f9794ab-4c1c-43a0-8657-5b1321d5d6c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: e155c2fe-6d5a-4b4f-bc63-af47d2b53ffe kind: Event lastTimestamp: "2026-04-24T21:50:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:11Z" name: error-404-isvc-1dabb-predictor.18a9695e47c88f2e namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 15916a0f-ac5c-457a-9a07-0042b532d90c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: e155c2fe-6d5a-4b4f-bc63-af47d2b53ffe kind: Event lastTimestamp: "2026-04-24T21:50:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:11Z" name: error-404-isvc-1dabb-predictor.18a9695e482b3990 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 152e1d4a-d249-4d14-8d8e-3afebed016ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1dabb namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 18eb6706-13fe-44de-98a9-27084687a10b kind: Event lastTimestamp: "2026-04-24T21:49:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:54Z" name: error-404-isvc-1dabb.18a96953cb41f88d namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 6e041e79-c62d-422e-a87d-5bf3fd5ed73e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1dabb namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 18eb6706-13fe-44de-98a9-27084687a10b kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-1dabb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1dabb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:56Z" name: error-404-isvc-1dabb.18a96953cba67161 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: d077ec3d-5476-4f2e-ab9b-86a946978327 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1dabb namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 18eb6706-13fe-44de-98a9-27084687a10b kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1dabb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:56Z" name: error-404-isvc-1dabb.18a96953cba6ccf9 namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: 795f7391-e311-431b-8648-6b8411425e24 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 18eb6706-13fe-44de-98a9-27084687a10b kind: Event lastTimestamp: "2026-04-24T21:49:54Z" message: InferenceService [error-404-isvc-1dabb] is Ready metadata: creationTimestamp: "2026-04-24T21:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:54Z" name: error-404-isvc-1dabb.18a969612a396fb6 namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: a3188dcb-be11-4657-b986-7837b68ad075 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:49:36.325856Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:49:36Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d0bd6b320 namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: ddd2100b-6e36-4d87-9877-db07d34dcd22 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1ec3f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d1cdc9483 namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 4a8a079a-0f94-432d-a682-76d43bda07a4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:49:37Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:49:37Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d495f8b82 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: 1289c590-317b-4c9f-855d-c431ac909d8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:49:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:37Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d4bb49f18 namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 3b076e58-4290-45c9-8529-05821201ee16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:49:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:37Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d4f21555d namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: 3e5197ae-a891-4fdb-b37f-f4bfdd1e2cb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:49:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:37Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d4f948710 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 71dd65bf-e0c7-40be-afe2-6db20e6157c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:49:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:37Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d4fa23798 namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 130a7457-9853-471c-9b65-6656a2b48b94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:49:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:37Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d5253e85d namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 055164da-f8c4-44ea-a4ae-ebf9b1166c16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:49:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:37Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d52c4a917 namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: 3247aab3-5d0a-465d-964e-cf08aaa9d1af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:50:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:15Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a9695d74e40c3b namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: 84f57403-a59b-4721-950b-82e71358cd2c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:51Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a969de3e6db859 namespace: kserve-ci-e2e-test resourceVersion: "25108" uid: 97f6c274-870e-4b87-a0d6-c6c423422ff4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 22ee0cfe-06f5-47f0-99f6-a77f434afb7c kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:51Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx.18a969de3e6e56b9 namespace: kserve-ci-e2e-test resourceVersion: "25109" uid: 47c12d30-baa0-481a-8fa1-a262005905e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1ec3f-predictor-5c44fc5cc6 namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: eae9b4a1-c6a7-43a6-a046-ed048fd5c825 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: 'Created pod: error-404-isvc-1ec3f-predictor-5c44fc5cc6-svnjx' metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:36Z" name: error-404-isvc-1ec3f-predictor-5c44fc5cc6.18a9695d0b75adab namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: b71fa12b-8af8-4843-9aa9-62b1ace0c118 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: af066472-0da5-4224-8bbe-581ea8c84798 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Scaled up replica set error-404-isvc-1ec3f-predictor-5c44fc5cc6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:36Z" name: error-404-isvc-1ec3f-predictor.18a9695d09d33969 namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: 12fa9591-9089-4eea-b75a-6e5dcf486fa3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 77206fb3-201d-4800-9fe2-25839f11a431 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:21Z" name: error-404-isvc-1ec3f-predictor.18a96960885b0bcd namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: bb8ee917-6ff5-46e9-90d6-c4fd63fb5d53 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 77206fb3-201d-4800-9fe2-25839f11a431 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:21Z" name: error-404-isvc-1ec3f-predictor.18a9696088bf905a namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 0bb1e56e-8104-46e8-99e2-8396f99f0b25 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 77206fb3-201d-4800-9fe2-25839f11a431 kind: Event lastTimestamp: "2026-04-24T21:50:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:51Z" name: error-404-isvc-1ec3f-predictor.18a96964075fae2e namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: 054f7743-6257-46f4-958d-77fcb94d35a2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 77206fb3-201d-4800-9fe2-25839f11a431 kind: Event lastTimestamp: "2026-04-24T21:50:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:51Z" name: error-404-isvc-1ec3f-predictor.18a9696407c6f766 namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: fbda7d69-aaef-46b8-8f82-07662fe1a00a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 4b2b41d0-6123-42a4-ac8e-646d089eb6b5 kind: Event lastTimestamp: "2026-04-24T21:50:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:25Z" name: error-404-isvc-1ec3f.18a9695d09aa793b namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: c55eb505-3dfb-494a-a6ab-fde919c15a59 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 4b2b41d0-6123-42a4-ac8e-646d089eb6b5 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-1ec3f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ec3f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:36Z" name: error-404-isvc-1ec3f.18a9695d0a023321 namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 9d5865a8-2163-4681-8aff-9d63561c2cb4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 4b2b41d0-6123-42a4-ac8e-646d089eb6b5 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ec3f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:36Z" name: error-404-isvc-1ec3f.18a9695d0a027c28 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 4cc3a728-4e67-478d-a4d3-50a7d6b9e0e6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: 4b2b41d0-6123-42a4-ac8e-646d089eb6b5 kind: Event lastTimestamp: "2026-04-24T21:50:25Z" message: InferenceService [error-404-isvc-1ec3f] is Ready metadata: creationTimestamp: "2026-04-24T21:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:25Z" name: error-404-isvc-1ec3f.18a969686c6e5d7e namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 24ea87c5-2bc3-4d37-bbec-7621364e10f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:47:46.240036Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2525b-predictor-76f74d576d-kh277 to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:47:46Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a969436a366301 namespace: kserve-ci-e2e-test resourceVersion: "19419" uid: cee102df-4ce2-4598-90f5-d5048fe60c5b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-2525b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:46Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a9694374e40183 namespace: kserve-ci-e2e-test resourceVersion: "19429" uid: 7d02d324-c4b9-4756-ae4c-4c70de94369c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:47:47Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96943a838ba5f namespace: kserve-ci-e2e-test resourceVersion: "19446" uid: 791a449f-ea08-4d2e-bb9a-8222e7ac4f1c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96943a989d8db namespace: kserve-ci-e2e-test resourceVersion: "19449" uid: a0733f51-4258-4007-8d56-fcc7280b2b57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96943ac6ef0bc namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: baf1c828-aa0e-4dba-8835-94e47959fb95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96943acf2d7b7 namespace: kserve-ci-e2e-test resourceVersion: "19455" uid: 21e6078d-9aa4-4d3d-8f31-0fc431249be0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96943acfe46c0 namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: b6ad74be-c544-4db4-a594-0be17539a019 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96943b070bdce namespace: kserve-ci-e2e-test resourceVersion: "19458" uid: a3f92f8d-365f-4a38-97cf-e9ad71b3e15e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96943b0d29f4e namespace: kserve-ci-e2e-test resourceVersion: "19459" uid: 81025551-17ea-42ce-9ad6-af22916b9065 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:48:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:24Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96943fa498ab2 namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: 662588a9-31a2-4887-bf4f-0231437f7bcb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:56Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96953c8a6252a namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 43f3a091-e2cc-47b7-afd8-ed5fdc055626 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:56Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96953c8a7355d namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: 5fe2e6ad-03bd-4801-92f9-0c4df389d395 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2525b-predictor-76f74d576d-kh277 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 1dfe4c57-e300-4bce-88f5-2f3d90e17cc0 kind: Event lastTimestamp: "2026-04-24T21:48:59Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": dial tcp 10.134.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:59Z" name: error-404-isvc-2525b-predictor-76f74d576d-kh277.18a96954829264aa namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: 4bb19b1a-39f0-4c24-a9b7-65e3c09d1f77 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2525b-predictor-76f74d576d namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: b04dea76-a610-46f9-92de-c2cf6474a3f6 kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: 'Created pod: error-404-isvc-2525b-predictor-76f74d576d-kh277' metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:46Z" name: error-404-isvc-2525b-predictor-76f74d576d.18a9694369cea6db namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: bcd78190-f72c-4295-9576-62d09141abde reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 394f0c2f-8214-4fd7-8289-31ac541eb815 kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: Scaled up replica set error-404-isvc-2525b-predictor-76f74d576d from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:46Z" name: error-404-isvc-2525b-predictor.18a96943682dd5a5 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: f20c5b10-80a0-4344-a7e7-915d4183247a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 30c16fad-b604-4f56-bce5-31047367dcf1 kind: Event lastTimestamp: "2026-04-24T21:48:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:16Z" name: error-404-isvc-2525b-predictor.18a96946e6a7a539 namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: e32183b4-aaec-4716-ba32-ea21ed69d561 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 30c16fad-b604-4f56-bce5-31047367dcf1 kind: Event lastTimestamp: "2026-04-24T21:48:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:16Z" name: error-404-isvc-2525b-predictor.18a96946e709750a namespace: kserve-ci-e2e-test resourceVersion: "19699" uid: f539a26e-23cf-471b-a8d2-6271914bb141 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:48:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 30c16fad-b604-4f56-bce5-31047367dcf1 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:46Z" name: error-404-isvc-2525b-predictor.18a9694de461c70e namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 8a750a9c-3640-42d5-bfd1-3413dd32390a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:48:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 30c16fad-b604-4f56-bce5-31047367dcf1 kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:46Z" name: error-404-isvc-2525b-predictor.18a9694de4c83106 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 99e97982-bdbe-45ce-b5fb-fd98ff0ad884 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2525b namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: ab15019e-5b07-4680-b6a5-9d7f6df167fe kind: Event lastTimestamp: "2026-04-24T21:48:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:34Z" name: error-404-isvc-2525b.18a96943680dc307 namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: d5980481-91eb-40e9-9ea9-43f9858a6c55 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2525b namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: ab15019e-5b07-4680-b6a5-9d7f6df167fe kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-2525b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2525b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:46Z" name: error-404-isvc-2525b.18a9694368afaf62 namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: f1983c09-b6c5-438e-a8f7-fbe324797237 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2525b namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: ab15019e-5b07-4680-b6a5-9d7f6df167fe kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2525b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:46Z" name: error-404-isvc-2525b.18a9694368affae8 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 920eb16f-1c8e-4efc-8550-5cb6f3677a8d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2525b namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: ab15019e-5b07-4680-b6a5-9d7f6df167fe kind: Event lastTimestamp: "2026-04-24T21:48:34Z" message: InferenceService [error-404-isvc-2525b] is Ready metadata: creationTimestamp: "2026-04-24T21:48:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:34Z" name: error-404-isvc-2525b.18a9694eb589d48a namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: 6d172e88-c21f-44de-825d-cc0b17a14e8e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:59:31.839926Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25919" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:59:31Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a969e7b33e3e2e namespace: kserve-ci-e2e-test resourceVersion: "25924" uid: 29861348-3a5b-4669-acc2-7f0859cbeacc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Add eth0 [10.134.0.55/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:59:32Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a969e7ccb7dae3 namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: a50f1a1f-c104-4135-bb04-4f1c2fe4dff2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a969e7ce810fe9 namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: df7770b1-2782-4d40-9010-2b1dfcd498d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a969e7d14c1e1b namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: ea33d070-034e-4b87-b4a4-26698c212865 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a969e7d1c09113 namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: 63f381a7-1e31-47a2-8498-7dadb9f8d897 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a969e7d1ccd3e9 namespace: kserve-ci-e2e-test resourceVersion: "25945" uid: 19c4a714-061f-4a3f-b4fa-7963da8b8ec6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a969e7d5bdb71e namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: f0442234-bb3f-4ca3-ba04-a56de744ea5b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a969e7d62d7107 namespace: kserve-ci-e2e-test resourceVersion: "25948" uid: 3cc27c7b-c5e3-4339-b119-9db4deff2234 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:59:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T22:08:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.55:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:49Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a969e8045efb3f namespace: kserve-ci-e2e-test resourceVersion: "30073" uid: 13c6999a-f0d8-41fd-8962-914d0736f931 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T22:08:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:46Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a96a68d21961df namespace: kserve-ci-e2e-test resourceVersion: "30047" uid: 0cd66c91-31a0-49bd-9419-abc9548e6527 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T22:08:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:46Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a96a68d219eaaf namespace: kserve-ci-e2e-test resourceVersion: "30048" uid: 096eb3a4-35a3-4b4b-af11-4c6befbe4e82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 19adae02-71e4-493f-b151-31853b1d445d kind: Event lastTimestamp: "2026-04-24T22:08:49Z" message: 'Readiness probe failed: Get "https://10.134.0.55:8643/healthz": dial tcp 10.134.0.55:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:49Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj.18a96a6978b8ff37 namespace: kserve-ci-e2e-test resourceVersion: "30071" uid: 26bac51b-a161-47a5-a494-70ab7bd010af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-600f1-predictor-7bcf7b8fd6 namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: 35548311-6a82-42b3-8d96-17eba814303b kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: 'Created pod: error-404-isvc-600f1-predictor-7bcf7b8fd6-f6xnj' metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:31Z" name: error-404-isvc-600f1-predictor-7bcf7b8fd6.18a969e7b2831371 namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: c614106c-e66a-4bbd-b5e3-973877d5393b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: c55c2c0c-26a1-4455-a4dd-b7b561e52f60 kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: Scaled up replica set error-404-isvc-600f1-predictor-7bcf7b8fd6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:31Z" name: error-404-isvc-600f1-predictor.18a969e7af1579bf namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: f3c83ad7-fce2-42a7-a443-28b39e676f14 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:59:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 1a76199e-2f98-4c4f-ba12-72d4c8d6c58d kind: Event lastTimestamp: "2026-04-24T22:00:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:59:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:01Z" name: error-404-isvc-600f1-predictor.18a969eb2d9d7b4f namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: 16a488e8-ed06-4552-a844-e66fa057443c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:59:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 1a76199e-2f98-4c4f-ba12-72d4c8d6c58d kind: Event lastTimestamp: "2026-04-24T22:00:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:59:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:01Z" name: error-404-isvc-600f1-predictor.18a969eb2deb6305 namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: 9bfbb04d-2861-45a5-b3b9-4d19e7a15d2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:00:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 1a76199e-2f98-4c4f-ba12-72d4c8d6c58d kind: Event lastTimestamp: "2026-04-24T22:00:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:46Z" name: error-404-isvc-600f1-predictor.18a969f22bc7ac62 namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: e8d33037-820f-495b-94a4-2eca1bc05887 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:00:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 1a76199e-2f98-4c4f-ba12-72d4c8d6c58d kind: Event lastTimestamp: "2026-04-24T22:00:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:46Z" name: error-404-isvc-600f1-predictor.18a969f22c4bc8c0 namespace: kserve-ci-e2e-test resourceVersion: "26710" uid: e4f25539-9590-4726-81c5-790191a61ddb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-600f1 namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 371f10be-e3a5-4e77-befd-e22213c26889 kind: Event lastTimestamp: "2026-04-24T22:00:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:00:19Z" name: error-404-isvc-600f1.18a969e7aed6d6ff namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: 5244d2c3-6342-4904-92f0-5ffa07e77641 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-600f1 namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 371f10be-e3a5-4e77-befd-e22213c26889 kind: Event lastTimestamp: "2026-04-24T22:00:19Z" message: InferenceService [error-404-isvc-600f1] is Ready metadata: creationTimestamp: "2026-04-24T22:00:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:00:19Z" name: error-404-isvc-600f1.18a969f2c357f1bf namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 8dc25063-8589-498d-a200-3b2f233c2c05 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:11.251959Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:11Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cb8287eb7c namespace: kserve-ci-e2e-test resourceVersion: "15582" uid: 5b76d656-9e22-4904-abc1-036ea23f9999 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7fc99-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cb923bc3a1 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: f16500b2-f7cb-43fa-aa76-60824d316496 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:39:12Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:12Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cbbfc82963 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 66e72fcf-10bf-4d18-81b2-a134da06c270 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:39:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:12Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cbc124dce3 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: ea454743-e53e-426b-9f07-e01beb3ae5cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:39:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:12Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cbc4812c78 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: 5d41942b-e776-46dc-b444-7cd0b5e3aa34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:39:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:12Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cbc4f23bb4 namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: 57cea817-e4bb-4c53-98e2-70f4b1c8465f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:39:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:12Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cbc4fe65cb namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 2ebd4ad4-36dc-4c1e-a4cf-77a6df66b637 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:39:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:12Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cbc7d7f32b namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 46e2a70b-ca5a-41d5-89f9-347b11c07f34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:39:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:12Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cbc849896e namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: 33bf431e-507a-4e19-9468-0acdf89078f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:48:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:29Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a968cc1fa6d020 namespace: kserve-ci-e2e-test resourceVersion: "19937" uid: 0680603d-e97b-40c3-bb29-b7008187e8bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a9694caf7fc7ea namespace: kserve-ci-e2e-test resourceVersion: "19848" uid: 701cbce8-89d2-4ebb-930f-1ee4bc939d9a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a9694caf805741 namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: 565112dc-98ae-4483-8e33-3119f6117743 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: aa940d38-3c6c-4e99-8e9b-95aa2b971edb kind: Event lastTimestamp: "2026-04-24T21:48:29Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:29Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n.18a9694d9400b434 namespace: kserve-ci-e2e-test resourceVersion: "19936" uid: 627737bc-0c2d-4a7f-814c-41df60e88328 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7fc99-predictor-5fcd68d6fb namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: e76464c2-a9f4-4718-9fa5-1c5bed09d908 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: 'Created pod: error-404-isvc-7fc99-predictor-5fcd68d6fb-tw55n' metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:11Z" name: error-404-isvc-7fc99-predictor-5fcd68d6fb.18a968cb821c03bd namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 72be4e7c-0052-4176-8b5a-83b8d55f8a62 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7fc99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: 085b95fb-bacc-43a4-8ca2-1c4507185bc7 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Scaled up replica set error-404-isvc-7fc99-predictor-5fcd68d6fb from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:11Z" name: error-404-isvc-7fc99-predictor.18a968cb80241a97 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: ec25245f-5376-4ff2-a838-746f1c3fe94f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7fc99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: ae48c580-f194-4256-bdcf-5ab76d71e1de kind: Event lastTimestamp: "2026-04-24T21:39:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:41Z" name: error-404-isvc-7fc99-predictor.18a968cefe8f7508 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: e7c8daaf-2ef2-4f3c-baf6-8f6deacf36f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7fc99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: ae48c580-f194-4256-bdcf-5ab76d71e1de kind: Event lastTimestamp: "2026-04-24T21:39:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:41Z" name: error-404-isvc-7fc99-predictor.18a968cefef74c70 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 202ebff3-78d5-49ee-97a5-79acc7917e9d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7fc99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: ae48c580-f194-4256-bdcf-5ab76d71e1de kind: Event lastTimestamp: "2026-04-24T21:40:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:11Z" name: error-404-isvc-7fc99-predictor.18a968d5fc9e7946 namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: ec522a61-2576-4ac8-9b73-7ec9dd936777 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7fc99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: ae48c580-f194-4256-bdcf-5ab76d71e1de kind: Event lastTimestamp: "2026-04-24T21:40:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:11Z" name: error-404-isvc-7fc99-predictor.18a968d5fd0ce7c6 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 47c949e6-b9c5-458a-8ca1-2558e9e25b1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7fc99 namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: 07104910-82e4-4168-beac-da17a5893852 kind: Event lastTimestamp: "2026-04-24T21:40:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:00Z" name: error-404-isvc-7fc99.18a968cb7ffda796 namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: bef3a4c5-5cc8-4702-96c4-2ea8b3df0908 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7fc99 namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: 07104910-82e4-4168-beac-da17a5893852 kind: Event lastTimestamp: "2026-04-24T21:39:59Z" message: InferenceService [error-404-isvc-7fc99] is Ready metadata: creationTimestamp: "2026-04-24T21:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:59Z" name: error-404-isvc-7fc99.18a968d6db641b74 namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: c3e070d0-1a02-4c0a-8b7c-4124f93b80a1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:37:01.203765Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-914c2-predictor-78956bf458-s7292 to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:37:01Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968ad3b0f48fb namespace: kserve-ci-e2e-test resourceVersion: "13968" uid: 00ed51a4-c4fa-4d97-b630-8f38d7d73e4e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-914c2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:01Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968ad4344b2c2 namespace: kserve-ci-e2e-test resourceVersion: "13979" uid: 12dcc281-ddb7-4510-b646-f1d640081c85 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:02Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:37:02Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968ad7885500b namespace: kserve-ci-e2e-test resourceVersion: "14033" uid: adba2e55-aa92-4311-bd4e-d63adb8f7947 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:02Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" metadata: creationTimestamp: "2026-04-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:02Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968ad7a39cf7f namespace: kserve-ci-e2e-test resourceVersion: "14036" uid: d1852861-5cef-41ab-8ca8-e15f438ab7e2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:18Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" in 15.94s (15.94s including waiting). Image size: 1334716521 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:18Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968b1306232a7 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 41a4b962-1c44-4d2c-9554-0709bd422f55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:18Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968b134a2a65d namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 1f81ca58-5532-4c32-a5d3-f8ed4735dd87 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:18Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968b13515a43b namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: e1e5d49b-df7e-4508-90d8-bb706b10bceb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:18Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968b13523e43b namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 8c2efdbc-5a45-41a2-b738-fe83ee9810fe reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:20Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.636s (2.636s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:20Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968b1d2497fc1 namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 99de5701-7bbb-4878-9df9-30e5adfa727b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:21Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968b1d79a5c85 namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: c63e652c-1546-489c-9959-b15f8b921b6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:21Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968b1d80f34b2 namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: ed0443a0-8a5b-48f9-971d-f570673e2be8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:37:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:58Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968b22ffed5fe namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: 7ad4227c-b92f-4708-b8fe-01d3553038b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:31Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968c22b670324 namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 77f4f2ee-a9be-4312-aab7-2c9bad0d01da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:31Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968c22b67d35e namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: f28914ac-4bca-4659-b329-853e12ac3389 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-914c2-predictor-78956bf458-s7292 namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: 9476e4c7-3607-4bc3-90dd-ed599afc66d8 kind: Event lastTimestamp: "2026-04-24T21:38:33Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:33Z" name: error-404-isvc-914c2-predictor-78956bf458-s7292.18a968c2b8168225 namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 1a0adfdb-13bf-4f65-851f-7f0c8239f1cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-914c2-predictor-78956bf458 namespace: kserve-ci-e2e-test resourceVersion: "13935" uid: 6df40881-8269-49cf-923f-2a5dbdf28520 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: 'Created pod: error-404-isvc-914c2-predictor-78956bf458-s7292' metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:01Z" name: error-404-isvc-914c2-predictor-78956bf458.18a968ad3a859186 namespace: kserve-ci-e2e-test resourceVersion: "13966" uid: 1e428d5b-73cb-4418-aed4-cf23f0fb452f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: ce734cd9-b624-4782-8066-21a028cc0f5e kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: Scaled up replica set error-404-isvc-914c2-predictor-78956bf458 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:01Z" name: error-404-isvc-914c2-predictor.18a968ad344ffb57 namespace: kserve-ci-e2e-test resourceVersion: "13938" uid: 45501e4e-f57d-46d5-8db1-f8ef0ba53b5f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13934" uid: 445b703d-9bd9-423e-897d-af4f6b8c7f6c kind: Event lastTimestamp: "2026-04-24T21:37:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:46Z" name: error-404-isvc-914c2-predictor.18a968b0b321213d namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 14283070-1a15-4adc-b3fb-109d90d572a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13934" uid: 445b703d-9bd9-423e-897d-af4f6b8c7f6c kind: Event lastTimestamp: "2026-04-24T21:37:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:46Z" name: error-404-isvc-914c2-predictor.18a968b0b38a6f93 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 10d3c63e-bf81-4f4e-b016-ce27d0af2166 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:38:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: 445b703d-9bd9-423e-897d-af4f6b8c7f6c kind: Event lastTimestamp: "2026-04-24T21:38:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:16Z" name: error-404-isvc-914c2-predictor.18a968bb2fd75884 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 66ef97d6-cbc4-43c1-8c7b-9310d826fe36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:38:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: 445b703d-9bd9-423e-897d-af4f6b8c7f6c kind: Event lastTimestamp: "2026-04-24T21:38:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:16Z" name: error-404-isvc-914c2-predictor.18a968bb304c9feb namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: 433eefa2-e91d-49ec-9a38-2b2aef279160 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-914c2 namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: e42695c9-5d00-40b7-89af-27bbd7193ebf kind: Event lastTimestamp: "2026-04-24T21:38:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:08Z" name: error-404-isvc-914c2.18a968ad342d0f80 namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: eec5f4a0-70b2-45dd-ba7a-22f0a9f400b2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-914c2 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: e42695c9-5d00-40b7-89af-27bbd7193ebf kind: Event lastTimestamp: "2026-04-24T21:38:08Z" message: InferenceService [error-404-isvc-914c2] is Ready metadata: creationTimestamp: "2026-04-24T21:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:08Z" name: error-404-isvc-914c2.18a968bcebd29227 namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: aebf3a19-f354-4932-95be-0d7429ecafbe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:58:21.433007Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24761" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a4170-predictor-59dc6578db-4k6pg to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969d74ea9e4ae namespace: kserve-ci-e2e-test resourceVersion: "24767" uid: e97d68b9-8dba-4875-9429-413746b1d40c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24769" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969d767a6b88b namespace: kserve-ci-e2e-test resourceVersion: "24780" uid: 04a2e2e3-c320-4bff-aa4b-f881e7c00269 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969d7692496cf namespace: kserve-ci-e2e-test resourceVersion: "24782" uid: 2b7cf1eb-397d-4de2-8cd8-8b07794b4a53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969d76bf8e5cf namespace: kserve-ci-e2e-test resourceVersion: "24783" uid: ee18a954-c2c7-414b-aa79-4e1beb8ee60d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969d76c6bca80 namespace: kserve-ci-e2e-test resourceVersion: "24784" uid: 2cd6e7cc-8cbd-4d0e-97e7-4d9dea3265a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969d76c76e7ea namespace: kserve-ci-e2e-test resourceVersion: "24785" uid: dfb8e9bf-97b6-46bb-8f14-8340404a1d95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969d76fbc3aa0 namespace: kserve-ci-e2e-test resourceVersion: "24786" uid: 863c0942-dc22-416d-a8a3-94af15dbfc4b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969d7702ffe43 namespace: kserve-ci-e2e-test resourceVersion: "24787" uid: f70b7ed7-b4c4-4cd4-b309-31f74f14d904 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:58:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:58:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:59Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969d7e201402c namespace: kserve-ci-e2e-test resourceVersion: "25333" uid: 4a7446db-aafa-4ad9-a973-f16810072e07 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:31Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969e7afbc5003 namespace: kserve-ci-e2e-test resourceVersion: "25917" uid: 03811bbe-c95a-46d1-93d3-1b6fe0708e30 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:31Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969e7afbcedf5 namespace: kserve-ci-e2e-test resourceVersion: "25918" uid: 929423a4-3ba8-4e51-921f-61d725e1eacc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 8cfe76c8-2cae-4af0-baca-9c7e7c62c3a2 kind: Event lastTimestamp: "2026-04-24T21:59:34Z" message: 'Readiness probe failed: Get "https://10.134.0.51:8643/healthz": dial tcp 10.134.0.51:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:34Z" name: error-404-isvc-a4170-predictor-59dc6578db-4k6pg.18a969e86a598ba5 namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: ed2570b7-d810-436a-8661-48941a5cce25 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a4170-predictor-59dc6578db namespace: kserve-ci-e2e-test resourceVersion: "24750" uid: 7718903c-8e47-4efd-92d7-fc1ed6ec3171 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: 'Created pod: error-404-isvc-a4170-predictor-59dc6578db-4k6pg' metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor-59dc6578db.18a969d74d1988ca namespace: kserve-ci-e2e-test resourceVersion: "24762" uid: 948b28bc-f93a-4808-bd29-6996b20ab5e4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24746" uid: 2a2a269c-e1dc-4b5c-905c-64263785cb76 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Scaled up replica set error-404-isvc-a4170-predictor-59dc6578db from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170-predictor.18a969d74a08d690 namespace: kserve-ci-e2e-test resourceVersion: "24753" uid: b4596d4e-c932-4f91-ab7c-b59253b2fac3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24749" uid: 8a6f2709-abdf-4ba0-8aea-36a62610dd14 kind: Event lastTimestamp: "2026-04-24T21:58:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:36Z" name: error-404-isvc-a4170-predictor.18a969dac8ae3590 namespace: kserve-ci-e2e-test resourceVersion: "24926" uid: 9eb5d903-2b29-4527-97c8-326aeafc21fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24749" uid: 8a6f2709-abdf-4ba0-8aea-36a62610dd14 kind: Event lastTimestamp: "2026-04-24T21:58:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:36Z" name: error-404-isvc-a4170-predictor.18a969dac9192b75 namespace: kserve-ci-e2e-test resourceVersion: "24928" uid: b11665a8-62c9-4bc8-8a76-352229fa29d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24927" uid: 8a6f2709-abdf-4ba0-8aea-36a62610dd14 kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:21Z" name: error-404-isvc-a4170-predictor.18a969de47d48c79 namespace: kserve-ci-e2e-test resourceVersion: "25726" uid: 7937a31f-4070-4ae9-8c22-8439e652a7bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24927" uid: 8a6f2709-abdf-4ba0-8aea-36a62610dd14 kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:21Z" name: error-404-isvc-a4170-predictor.18a969de48221e0c namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: 69e48744-0e8d-4fea-ad92-ec9036587bef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a4170 namespace: kserve-ci-e2e-test resourceVersion: "24723" uid: 986b5d5a-a5f5-48a2-9f48-a0b9e5902f0a kind: Event lastTimestamp: "2026-04-24T21:59:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:10Z" name: error-404-isvc-a4170.18a969d74a0d7ffe namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: 2bedb04a-7a96-4f9a-aada-c57c4e9fba1a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a4170 namespace: kserve-ci-e2e-test resourceVersion: "24723" uid: 986b5d5a-a5f5-48a2-9f48-a0b9e5902f0a kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-a4170": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a4170": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170.18a969d74adb19b0 namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: 95658062-3509-424f-b939-2627263fc867 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a4170 namespace: kserve-ci-e2e-test resourceVersion: "24723" uid: 986b5d5a-a5f5-48a2-9f48-a0b9e5902f0a kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a4170": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:58:21Z" name: error-404-isvc-a4170.18a969d74adb8638 namespace: kserve-ci-e2e-test resourceVersion: "24758" uid: 3d614616-8425-4b77-bf12-9c2ff56f276a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a4170 namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: 986b5d5a-a5f5-48a2-9f48-a0b9e5902f0a kind: Event lastTimestamp: "2026-04-24T21:59:10Z" message: InferenceService [error-404-isvc-a4170] is Ready metadata: creationTimestamp: "2026-04-24T21:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:10Z" name: error-404-isvc-a4170.18a969e2a0422843 namespace: kserve-ci-e2e-test resourceVersion: "25607" uid: 971f1064-0f5d-4a88-94cb-df1093079eb2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:48:26.121234Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b9878-predictor-65869744dd-gsdxb to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9694cb3511ce2 namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: 3cd9ff3f-017b-4a39-a824-25bdd5a89689 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9694ccd23f689 namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: b19e5663-7243-4b65-afaf-b2275d1bad7c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9694ccf5778cd namespace: kserve-ci-e2e-test resourceVersion: "19887" uid: 8f22e9dc-491b-4d83-a93a-b475f94c26d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9694cd2516ab9 namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: fed97ac6-5d3d-4fd0-a715-3fb1baa99e38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9694cd2c3324b namespace: kserve-ci-e2e-test resourceVersion: "19889" uid: 3c32b760-4134-41d5-8f46-2dc901e44fe5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9694cd2d05e25 namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 55773d44-dbe6-440e-9dd5-1830f99c754f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9694cd656ce7b namespace: kserve-ci-e2e-test resourceVersion: "19891" uid: 7e174146-42ea-4e24-a6ef-478149d83d0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9694cd6c8bdfb namespace: kserve-ci-e2e-test resourceVersion: "19892" uid: 37d66ab5-981e-47c8-bab6-a30c818d51f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:49:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:03Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9694d183337f5 namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: e4e53c15-0f63-4f67-baf7-bcea951824d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9695d01dac877 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 23105c75-5c10-402f-99d0-8322aef2bae3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9695d01db0a41 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 1b55daa8-010d-4a88-82c4-717f2f4dd4b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9878-predictor-65869744dd-gsdxb namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 9a2cafa0-f418-40ed-a22a-f8abd1a4016b kind: Event lastTimestamp: "2026-04-24T21:49:38Z" message: 'Readiness probe failed: Get "https://10.134.0.45:8643/healthz": dial tcp 10.134.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:38Z" name: error-404-isvc-b9878-predictor-65869744dd-gsdxb.18a9695da0421059 namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: 978e16a8-d582-4686-ac31-c4fd415860fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b9878-predictor-65869744dd namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 78d9dd44-d79b-4f31-ad72-b32fe3587cab kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: 'Created pod: error-404-isvc-b9878-predictor-65869744dd-gsdxb' metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor-65869744dd.18a9694cb2d7d943 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 91135b55-3956-4e01-97f7-3ffb9888bf72 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 7443f151-df19-4b5d-af60-a26d7e71644b kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: Scaled up replica set error-404-isvc-b9878-predictor-65869744dd from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878-predictor.18a9694cb123b2f3 namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: 30a760ef-871d-4407-ae2b-6b01e4456b1b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:48:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: e65f3899-9afe-4124-91e7-814bcda13b5e kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:56Z" name: error-404-isvc-b9878-predictor.18a96950305c126c namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: 155841b1-9f2c-4277-ad69-814b47fd6f78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:48:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: e65f3899-9afe-4124-91e7-814bcda13b5e kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:56Z" name: error-404-isvc-b9878-predictor.18a9695031225215 namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: bab55fb2-b73f-4050-91e8-2e14f5e062d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: e65f3899-9afe-4124-91e7-814bcda13b5e kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: error-404-isvc-b9878-predictor.18a969572e958994 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 812b8be2-534c-493b-ad4a-9b841fe3d308 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: e65f3899-9afe-4124-91e7-814bcda13b5e kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: error-404-isvc-b9878-predictor.18a969572f30151e namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: ba471e41-ea63-449e-a443-8b94145ae5e8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9878 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 85f35ea6-d101-4a80-b524-01f9e31c5359 kind: Event lastTimestamp: "2026-04-24T21:49:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:13Z" name: error-404-isvc-b9878.18a9694cb0fcc6ba namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: abd9209d-b9af-4d62-8b07-4659b3ebcc75 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9878 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 85f35ea6-d101-4a80-b524-01f9e31c5359 kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-b9878": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b9878": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878.18a9694cb1a69dcc namespace: kserve-ci-e2e-test resourceVersion: "19859" uid: 88e2f601-117e-4c73-b40f-3277b9e68625 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9878 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 85f35ea6-d101-4a80-b524-01f9e31c5359 kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b9878": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:26Z" name: error-404-isvc-b9878.18a9694cb1a700b4 namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: 566d5750-1740-4b30-a55b-b0256ad2aa5b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9878 namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: 85f35ea6-d101-4a80-b524-01f9e31c5359 kind: Event lastTimestamp: "2026-04-24T21:49:13Z" message: InferenceService [error-404-isvc-b9878] is Ready metadata: creationTimestamp: "2026-04-24T21:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:13Z" name: error-404-isvc-b9878.18a96957d30a1b6c namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 7c4f3631-6673-4b6d-b0b6-cfd5060344a2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:38:31.314857Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:38:31Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c236181a12 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 82780e6a-6811-4977-87a9-6f9a9cf5cbc8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c12fa-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:31Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c23f2aa0af namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: e35383ce-1886-4f13-8834-a2fdf81c2727 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:38:32Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c273e1b51c namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: 165d2a00-1f22-49d3-93c3-c7fe7d79ecc4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c2754004a7 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 139e4cfa-072d-4be9-925a-81dcfb06a146 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c279c9525b namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: 0dbc9af3-3466-4914-a3f4-f8748410f423 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c27a435ff0 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 98cb3637-34fc-4971-8cfb-00908015ff05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c27a50601e namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a6e14835-f842-4d77-9352-d37de0ed666e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c27d3cbee7 namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 0c150867-25e3-43ea-b49f-97bf3a548fd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c27db0f33d namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: 2493a65f-71d0-4f32-803a-1144d56a50ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:47:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:48Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a968c28ac0838c namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 3cd32cde-75ca-4320-a5c0-7a6fb128a429 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:46Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a96943653af834 namespace: kserve-ci-e2e-test resourceVersion: "19390" uid: 5296f3e5-90b8-44e6-b39d-f630dcd677c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:46Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a96943653b785e namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: dce2176a-5be6-45e7-ba82-88b3ab9dd6d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: e6b72a15-e99f-4e4c-98c3-6e87f0b7458d kind: Event lastTimestamp: "2026-04-24T21:47:48Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:48Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h.18a96943ff19cf27 namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: b75c5126-110f-4ca1-9f9e-cd2d677ebc8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c12fa-predictor-6c7dd5899d namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: 1689e8cb-5e73-441f-bc67-499d3370e41f kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: 'Created pod: error-404-isvc-c12fa-predictor-6c7dd5899d-64n9h' metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:31Z" name: error-404-isvc-c12fa-predictor-6c7dd5899d.18a968c2359e08bc namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: ca4433f1-0409-4003-a49c-35f4a6fbb968 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: b000b001-6405-43e5-a504-99ffc474d7c4 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: Scaled up replica set error-404-isvc-c12fa-predictor-6c7dd5899d from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:31Z" name: error-404-isvc-c12fa-predictor.18a968c2340273d8 namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: d504aee2-7f7a-4efa-8254-562cef0d0a33 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 14422150-13d8-4bb9-b340-09f15351e394 kind: Event lastTimestamp: "2026-04-24T21:39:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:16Z" name: error-404-isvc-c12fa-predictor.18a968c5b2a7e991 namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: ef96b434-fe54-4a87-9e1d-7468f2f2881c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 14422150-13d8-4bb9-b340-09f15351e394 kind: Event lastTimestamp: "2026-04-24T21:39:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:16Z" name: error-404-isvc-c12fa-predictor.18a968c5b31e7e27 namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: fb3edf8e-b0ef-48f9-9b37-c05cb4571055 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 14422150-13d8-4bb9-b340-09f15351e394 kind: Event lastTimestamp: "2026-04-24T21:39:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:46Z" name: error-404-isvc-c12fa-predictor.18a968d02f54bd7d namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: dbd93235-ada6-4227-82b7-4162553cf1fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 14422150-13d8-4bb9-b340-09f15351e394 kind: Event lastTimestamp: "2026-04-24T21:39:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:46Z" name: error-404-isvc-c12fa-predictor.18a968d02fb06d16 namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: b6fa97fc-4e93-436b-bab3-712043ce570e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c12fa namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 3a61c1f1-0d0e-4807-9198-67556ed6dabf kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:18Z" name: error-404-isvc-c12fa.18a968c2340ba032 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: b28cd561-1132-4a85-bd54-d13ef5de9640 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c12fa namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: 3a61c1f1-0d0e-4807-9198-67556ed6dabf kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: InferenceService [error-404-isvc-c12fa] is Ready metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:18Z" name: error-404-isvc-c12fa.18a968cd48755e28 namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: e03e679f-cc4a-4a72-96c1-5a8c61a3937e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:58:51.328157Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25130" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:58:51Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a969de448dbc27 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 237f1d3c-eb3b-40e5-a187-4d5e963518c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25140" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: Add eth0 [10.134.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:58:51Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a969de6ad70d6a namespace: kserve-ci-e2e-test resourceVersion: "25155" uid: 7581727a-c94a-4566-8fc3-07f7c33257bb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:51Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a969de6c445f0a namespace: kserve-ci-e2e-test resourceVersion: "25157" uid: e9a16efe-1a19-41b5-a8a0-f72107d80e1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a969de6f19aca6 namespace: kserve-ci-e2e-test resourceVersion: "25159" uid: 6e24a656-ff23-49f8-aa44-5e58de6283be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a969de6f8ffa74 namespace: kserve-ci-e2e-test resourceVersion: "25161" uid: e231b7f5-acdc-4cc5-90fc-e42c02b32c4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a969de6f9c11ee namespace: kserve-ci-e2e-test resourceVersion: "25163" uid: 31e6cf47-1a2b-4e56-a648-83953e9ff306 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a969de73b056f0 namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: 4ebb7f4c-8e7b-47f2-a93f-d4953d9f2d96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a969de742b7c68 namespace: kserve-ci-e2e-test resourceVersion: "25166" uid: 7999f967-fe60-4e42-8f6f-b028557cb3f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:58:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T21:59:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.53:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:29Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a969deaa4c7c3b namespace: kserve-ci-e2e-test resourceVersion: "25809" uid: 2a90d16f-663e-437c-80d9-56a92841549a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T22:16:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:16:10Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a96ad045f92f66 namespace: kserve-ci-e2e-test resourceVersion: "33140" uid: bfd10d98-6cb4-4bf6-b694-2c28b559f53d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 6ad1d5e7-a76a-4378-b25d-eaeeea0f959f kind: Event lastTimestamp: "2026-04-24T22:16:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:16:10Z" name: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6.18a96ad045f9d564 namespace: kserve-ci-e2e-test resourceVersion: "33141" uid: 874eebba-475a-4381-acf0-d48d116ec8df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f4deb-predictor-c578cbf66 namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: 2896c707-f161-4cd6-a633-f401bfaa612e kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: 'Created pod: error-404-isvc-f4deb-predictor-c578cbf66-dkmz6' metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:51Z" name: error-404-isvc-f4deb-predictor-c578cbf66.18a969de441e42e2 namespace: kserve-ci-e2e-test resourceVersion: "25131" uid: f3dd2978-f102-4d01-a76b-0f69dd30c150 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25117" uid: f80001f0-5051-4073-b70c-ddd09b5560d0 kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: Scaled up replica set error-404-isvc-f4deb-predictor-c578cbf66 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:51Z" name: error-404-isvc-f4deb-predictor.18a969de41f94561 namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: 83e93eb8-332d-484d-a1ad-5b51e31b4cf8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25118" uid: e4141845-bbca-4ead-bb63-701d5ab6671f kind: Event lastTimestamp: "2026-04-24T21:59:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:36Z" name: error-404-isvc-f4deb-predictor.18a969e1c15538ff namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: 91bc018f-701d-495d-abe3-9f71ce832a7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25118" uid: e4141845-bbca-4ead-bb63-701d5ab6671f kind: Event lastTimestamp: "2026-04-24T21:59:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:36Z" name: error-404-isvc-f4deb-predictor.18a969e1c25c7b20 namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 3318a494-41e3-4719-9fd7-37d3dec8844f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:59:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25561" uid: e4141845-bbca-4ead-bb63-701d5ab6671f kind: Event lastTimestamp: "2026-04-24T22:00:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:21Z" name: error-404-isvc-f4deb-predictor.18a969ec3f6f426c namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: 313f664e-8682-4c60-bfdf-d7cc47358396 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:59:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25561" uid: e4141845-bbca-4ead-bb63-701d5ab6671f kind: Event lastTimestamp: "2026-04-24T22:00:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:21Z" name: error-404-isvc-f4deb-predictor.18a969ec40027db5 namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: 2a52cb16-93e4-4d49-8077-0cd79cd0f6c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f4deb namespace: kserve-ci-e2e-test resourceVersion: "25103" uid: 7b1fb2a5-1ee1-4eb5-8704-7ca649e2e3da kind: Event lastTimestamp: "2026-04-24T21:59:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:39Z" name: error-404-isvc-f4deb.18a969de41a8f882 namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 0cd4aac2-7642-4fa5-a863-426c9852e6fb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f4deb namespace: kserve-ci-e2e-test resourceVersion: "25103" uid: 7b1fb2a5-1ee1-4eb5-8704-7ca649e2e3da kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-f4deb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f4deb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:58:51Z" name: error-404-isvc-f4deb.18a969de421e8a74 namespace: kserve-ci-e2e-test resourceVersion: "25125" uid: c9f3bf56-d520-4b56-8b0b-848bbe87aec0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f4deb namespace: kserve-ci-e2e-test resourceVersion: "25103" uid: 7b1fb2a5-1ee1-4eb5-8704-7ca649e2e3da kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f4deb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:58:51Z" name: error-404-isvc-f4deb.18a969de421ecfcb namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 0f5c140c-d3e9-40b8-9c1f-b9c5690fe381 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f4deb namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 7b1fb2a5-1ee1-4eb5-8704-7ca649e2e3da kind: Event lastTimestamp: "2026-04-24T21:59:39Z" message: InferenceService [error-404-isvc-f4deb] is Ready metadata: creationTimestamp: "2026-04-24T21:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:39Z" name: error-404-isvc-f4deb.18a969e965ca6a58 namespace: kserve-ci-e2e-test resourceVersion: "26044" uid: 2f30c605-2823-49e4-82fb-e3e2cf4be5d8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:37:00.916732Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13889" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-79868c757f-zc9fh to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-04-24T21:37:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:37:00Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968ad29f23da7 namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: da030cae-743c-41f1-bb52-c23122f4c4f4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:37:01Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968ad435614df namespace: kserve-ci-e2e-test resourceVersion: "13980" uid: 8c04e46d-dfb8-4099-935c-7e837a00b31c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:01Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968ad44d49625 namespace: kserve-ci-e2e-test resourceVersion: "13982" uid: e173af0a-d250-4079-bc9f-6f3ee5a56380 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" in 3.207s (3.207s including waiting). Image size: 299849138 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:04Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968ae03fd3e36 namespace: kserve-ci-e2e-test resourceVersion: "14069" uid: 20b07fb9-089e-48da-a636-1ebd6a52ca7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:04Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968ae0724c272 namespace: kserve-ci-e2e-test resourceVersion: "14070" uid: e4796d77-e272-458e-9599-3dd75d9e2449 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:04Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968ae07dc8205 namespace: kserve-ci-e2e-test resourceVersion: "14071" uid: 516605bd-ad59-478a-95bb-44ab74efca13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:08Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" metadata: creationTimestamp: "2026-04-24T21:37:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:08Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968aef1b1e423 namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: c1237da1-0f23-4822-8685-5f80b32a0db1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" in 15.351s (15.351s including waiting). Image size: 1560926130 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:23Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968b284b06646 namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: cda379e6-15c9-4117-84a3-9d01bbc35fed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:23Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968b2886bd3b4 namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: 93d5a246-e649-4b21-8993-849eb0f977f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:23Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968b288e52b8b namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: 00528950-dd06-4963-840d-fa476efb9a20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:23Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968b288f32e4e namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 2782e399-db93-4633-a4a7-ebdd11d8d96b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:26Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.442s (2.442s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:26Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968b31a83bf1e namespace: kserve-ci-e2e-test resourceVersion: "14319" uid: 1946353c-f570-4358-8562-55fde8eb1f7e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:26Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968b31ebabe21 namespace: kserve-ci-e2e-test resourceVersion: "14321" uid: 9fc59d23-799b-4ef8-bb7b-0fc37473c0bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:37:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:26Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968b31f316c6c namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 19cc7c2a-e123-43f8-99f8-ec39329ff0bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:39:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:13Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968b363762f5c namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 6739ac46-c605-4ea0-a033-73ce01a95ff3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968cb7b81da2e namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 5536bc99-a86b-4869-84bc-180350299dc0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968cb7b83242a namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: c2563c36-7e48-4714-a0b1-22076ca538c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: f4bdf6b1-e39b-4813-8d7c-e4b8a0bdafb3 kind: Event lastTimestamp: "2026-04-24T21:39:13Z" message: 'Readiness probe failed: Get "https://10.133.0.17:8643/healthz": dial tcp 10.133.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:13Z" name: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh.18a968cc11a5e966 namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: 516f2889-1438-4a5e-8274-ec601d13646c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-79868c757f namespace: kserve-ci-e2e-test resourceVersion: "13882" uid: 05ff3560-d6eb-4611-a7f4-f819f4978859 kind: Event lastTimestamp: "2026-04-24T21:37:00Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-79868c757f-zc9fh' metadata: creationTimestamp: "2026-04-24T21:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:00Z" name: isvc-sklearn-graph-1-predictor-79868c757f.18a968ad298cbbc5 namespace: kserve-ci-e2e-test resourceVersion: "13893" uid: 2006d43b-dfbb-48e0-935f-2430d5465bb2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13881" uid: f1648874-fb98-453e-a5d1-9be4cafcf356 kind: Event lastTimestamp: "2026-04-24T21:37:00Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-79868c757f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:00Z" name: isvc-sklearn-graph-1-predictor.18a968ad238de033 namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: eeb16d7f-1b80-42ac-9cfc-539c2be38e22 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: e432f5ef-e63a-4813-b43e-bfd65db936f9 kind: Event lastTimestamp: "2026-04-24T21:38:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:01Z" name: isvc-sklearn-graph-1-predictor.18a968b0a977352a namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: 714e93cb-8dc8-49f7-8b99-fbd7fe476240 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: e432f5ef-e63a-4813-b43e-bfd65db936f9 kind: Event lastTimestamp: "2026-04-24T21:38:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:01Z" name: isvc-sklearn-graph-1-predictor.18a968b0a9dfd398 namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 292913c4-e275-463e-b315-b44ea6fa78cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: e432f5ef-e63a-4813-b43e-bfd65db936f9 kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:01Z" name: isvc-sklearn-graph-1-predictor.18a968b7a828ddd1 namespace: kserve-ci-e2e-test resourceVersion: "15366" uid: 6d4c9637-810c-49bd-a96e-4b4784c27320 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: e432f5ef-e63a-4813-b43e-bfd65db936f9 kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:01Z" name: isvc-sklearn-graph-1-predictor.18a968b7a8957de8 namespace: kserve-ci-e2e-test resourceVersion: "15367" uid: a6ee4261-db2b-4ef7-aadb-929a162b74fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:37:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13866" uid: 406e994c-3e31-41f2-9d4f-5111ecdd4d23 kind: Event lastTimestamp: "2026-04-24T21:38:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:33Z" name: isvc-sklearn-graph-1.18a968ad29d0b654 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: d49907bb-4f2c-4f8d-8711-ff874ec300e0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 406e994c-3e31-41f2-9d4f-5111ecdd4d23 kind: Event lastTimestamp: "2026-04-24T21:38:33Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-24T21:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:33Z" name: isvc-sklearn-graph-1.18a968c2c94a1ff0 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: 97d6622b-b9eb-4374-845d-3573edbbacde reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:37:02.034857Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14019" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7758df598f-nldq8 to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-04-24T21:37:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:37:02Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968ad6c97701f namespace: kserve-ci-e2e-test resourceVersion: "14023" uid: 2526586e-af30-4f6c-b1f7-d1ebc6d5c9f4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14028" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:02Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:37:02Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968ad87e1bec9 namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: b8c8e4f8-dc38-49ce-b18a-d0cd5ceeb29a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:04Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" metadata: creationTimestamp: "2026-04-24T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:04Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968ae00e68cbf namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: ee6ed552-296b-4b4f-b2a2-f4f100ed2ac3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" in 322ms (322ms including waiting). Image size: 299849138 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:04Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968ae141e6dcf namespace: kserve-ci-e2e-test resourceVersion: "14074" uid: 8f6ff90e-d83e-400a-a7ca-ef491d66dc9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:04Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968ae16f387d0 namespace: kserve-ci-e2e-test resourceVersion: "14075" uid: 506d0964-26b2-49e3-a7f4-6d6c39dcc813 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:04Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968ae176ce538 namespace: kserve-ci-e2e-test resourceVersion: "14076" uid: 1244f3ac-8c26-474e-837f-c6d870cfd469 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:09Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" metadata: creationTimestamp: "2026-04-24T21:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:09Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968af2db08a07 namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: 88279352-4e9a-4f8e-8330-955782f4420e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1449" in 14.352s (14.352s including waiting). Image size: 1560926130 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:23Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968b2852e8ac3 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 455b1e42-dddd-4938-9574-21d9dba22d07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:23Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968b2888a1d5f namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 501fe888-e7bd-42fd-8fb9-3b7646526ed2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:24Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968b2891da9ae namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: f5900bbc-b697-4142-a138-effab9360655 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:24Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968b289299ebd namespace: kserve-ci-e2e-test resourceVersion: "14299" uid: d68a307a-3797-483d-8097-132fdded18f3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:26Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.432s (2.432s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:26Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968b31a2219fe namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: b71f16e3-2aab-40ed-a39c-ff3fff82aa60 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:26Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968b31e88c9c0 namespace: kserve-ci-e2e-test resourceVersion: "14320" uid: 294c2b9a-2e80-4ab2-a68b-f5087142d887 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:37:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:26Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968b31f146752 namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: ec2afd76-3d46-48fe-9510-e1e53e3400b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:39:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:13Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968b3637598af namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 2de968f2-4181-4d09-9df6-f315e47dad00 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968cb789aa836 namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: e4159a68-3955-4383-a6a4-e762bc4a95f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968cb789b4082 namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: b272d4e6-f85f-4762-9fae-e12d437bdd58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: f8e80edf-1330-461d-8e2e-52e2cf2c9b92 kind: Event lastTimestamp: "2026-04-24T21:39:13Z" message: 'Readiness probe failed: Get "https://10.133.0.18:8643/healthz": dial tcp 10.133.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:13Z" name: isvc-sklearn-graph-2-predictor-7758df598f-nldq8.18a968cc11a56139 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 4629d0ba-7197-4e74-bbac-3edfabcf1e60 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7758df598f namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: 1c0845a3-658f-4863-a913-39f95c7a8543 kind: Event lastTimestamp: "2026-04-24T21:37:02Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7758df598f-nldq8' metadata: creationTimestamp: "2026-04-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:02Z" name: isvc-sklearn-graph-2-predictor-7758df598f.18a968ad6c2a7b42 namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: a04e4645-f766-46de-b6fb-479db10ebd56 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14001" uid: 83e2c255-be55-4571-a3c8-d74919c732f4 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7758df598f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:01Z" name: isvc-sklearn-graph-2-predictor.18a968ad5605449f namespace: kserve-ci-e2e-test resourceVersion: "14004" uid: 8958b914-7864-4302-813f-9df49f6efc93 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: dfa6e83a-abd2-4c23-9acf-e4fffb1dfeb5 kind: Event lastTimestamp: "2026-04-24T21:38:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:01Z" name: isvc-sklearn-graph-2-predictor.18a968b0d49418e7 namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 45174220-e763-4133-9e00-10af5172c832 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: dfa6e83a-abd2-4c23-9acf-e4fffb1dfeb5 kind: Event lastTimestamp: "2026-04-24T21:38:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:01Z" name: isvc-sklearn-graph-2-predictor.18a968b0d4eedbb7 namespace: kserve-ci-e2e-test resourceVersion: "14637" uid: 298c8d51-9340-4f5a-af80-04057ead02e3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:37:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: dfa6e83a-abd2-4c23-9acf-e4fffb1dfeb5 kind: Event lastTimestamp: "2026-04-24T21:38:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:46Z" name: isvc-sklearn-graph-2-predictor.18a968b7d27eebbb namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 53509af3-aa12-4b4b-aa18-147d64dee2c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:37:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: dfa6e83a-abd2-4c23-9acf-e4fffb1dfeb5 kind: Event lastTimestamp: "2026-04-24T21:38:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:46Z" name: isvc-sklearn-graph-2-predictor.18a968b7d2edc182 namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: 1fd8db7d-483b-4606-bbd7-33e51017ab4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13972" uid: 95dc0a69-2485-4200-af67-4525bd1b5f82 kind: Event lastTimestamp: "2026-04-24T21:38:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:33Z" name: isvc-sklearn-graph-2.18a968ad55e4bbba namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 630bc46d-5e65-42f0-a684-fe9f2a6a2b5f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 95dc0a69-2485-4200-af67-4525bd1b5f82 kind: Event lastTimestamp: "2026-04-24T21:38:33Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-24T21:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:33Z" name: isvc-sklearn-graph-2.18a968c2c6e05506 namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: e849dc9d-9472-4fb1-bcb1-d2d18b553678 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:37:01.388444Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-6z8g6 to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:37:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968ad460ffa7b namespace: kserve-ci-e2e-test resourceVersion: "13987" uid: 270f552d-6f52-4508-a277-3874fe7d2f98 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13991" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:37:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968ad5f09f55a namespace: kserve-ci-e2e-test resourceVersion: "14013" uid: 5d3fde66-700e-45f3-b4f4-704df1f2d426 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968ad60adfad7 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 65ce5db7-bcdb-4669-a6a7-eb61b5b1db67 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:05Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b2a48068363ac4e806775467ef2dc5b6c858faecf49ae1f8bafc4262d036f97e" in 3.173s (3.173s including waiting). Image size: 299849138 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968ae1dd12880 namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: 135fd344-ad0f-4fe3-9870-26776c826181 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968ae20990ce0 namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: af1b78f4-3e39-448e-bd0c-71606332a699 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968ae2120223d namespace: kserve-ci-e2e-test resourceVersion: "14081" uid: 360cb7f1-172c-4fbe-981a-61334f84bcc7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:09Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T21:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:09Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968af108ab979 namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 12d07462-247a-4ef0-9415-0fb169a57fb5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:26Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.886s (17.886s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:26Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968b33aa728ac namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 7e075ef0-68f4-442b-b836-4a8846b99e0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968b33d9592b2 namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: bae565f3-7ad0-42bb-b953-38a68f2c05df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968b33df99aca namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: f4ee7867-8bbe-4995-ab70-80867981f7f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:27Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968b33e06766f namespace: kserve-ci-e2e-test resourceVersion: "14332" uid: 309736a0-24f1-4b7d-9be8-387cdf769e36 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:29Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.142s (2.142s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968b3bdc161f7 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 395ba699-c7fa-42fd-b947-9275282bc72c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968b3c10ab2c1 namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: 2ea020cd-1208-46ca-a50e-5e0e3979ef36 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:37:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968b3c16eff2a namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: a723f9a3-48ab-493f-ad56-05f122e4cc41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T21:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:38:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968b43461763d namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: 1e071f78-6cdd-4720-91fe-2080d5caa61c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968cb81844e65 namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 3539b8f1-f685-403e-ad41-c8fdee2583ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968cb8184fd09 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 94afade7-0c34-4c75-8ad7-2efcc87a03ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: a1ab7a10-0005-4d6c-a55d-b1d39e63c048 kind: Event lastTimestamp: "2026-04-24T21:39:12Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:12Z" name: isvc-xgboost-graph-predictor-669d8d6456-6z8g6.18a968cbb89c28f4 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 416e612b-c925-46b6-9ac5-350765a79f63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "13959" uid: 936120ba-b809-4807-9c1f-6521d97ac22f kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-6z8g6' metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:01Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a968ad457db4b6 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 309f7ca4-f4f1-4825-aa1b-4b496d0cb113 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13957" uid: d488d609-bbf1-45c9-b9d3-75b42a06e620 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:01Z" name: isvc-xgboost-graph-predictor.18a968ad3a2bd747 namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: 9c6d4c37-e3ba-4e34-bc19-6a37b99ec89f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:37:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13958" uid: 9c271580-71df-4f05-8b16-f25e2c04de06 kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:01Z" name: isvc-xgboost-graph-predictor.18a968b0b8bfc34a namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: 6ce328e6-c5b2-4312-826f-db2ee47b6dbe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:37:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13958" uid: 9c271580-71df-4f05-8b16-f25e2c04de06 kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:01Z" name: isvc-xgboost-graph-predictor.18a968b0b9314f6e namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: 2662b63e-4619-480f-aed7-9f0bb4e1a547 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:37:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 9c271580-71df-4f05-8b16-f25e2c04de06 kind: Event lastTimestamp: "2026-04-24T21:38:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:46Z" name: isvc-xgboost-graph-predictor.18a968b7b721161d namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: faa169f3-cc8d-4178-a03a-755128d10c40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:37:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 9c271580-71df-4f05-8b16-f25e2c04de06 kind: Event lastTimestamp: "2026-04-24T21:38:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:37:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:46Z" name: isvc-xgboost-graph-predictor.18a968b7b7b1ee65 namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: c0d0f79c-248b-4c95-affa-430f0871bdd4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: 87068590-f5cd-46aa-8584-c4a40d36f0e2 kind: Event lastTimestamp: "2026-04-24T21:38:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:37Z" name: isvc-xgboost-graph.18a968ad3a12b969 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 90e4aa0d-6119-4a69-84cf-78b380bafae5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 87068590-f5cd-46aa-8584-c4a40d36f0e2 kind: Event lastTimestamp: "2026-04-24T21:38:37Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-24T21:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:37Z" name: isvc-xgboost-graph.18a968c39811c749 namespace: kserve-ci-e2e-test resourceVersion: "15144" uid: 47ffb0d7-d139-410a-b41b-d76a20e175aa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:00.864810Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64c4bcbb69-xgd8t namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: 21fcd48e-105c-4d7a-be45-51cfdea3fb4d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-64c4bcbb69-xgd8t to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:00Z" name: model-chainer-64c4bcbb69-xgd8t.18a968c917689486 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: b72638eb-92e6-474e-8eeb-0fa37db89b65 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64c4bcbb69-xgd8t namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 21fcd48e-105c-4d7a-be45-51cfdea3fb4d kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:01Z" name: model-chainer-64c4bcbb69-xgd8t.18a968c9272e63a7 namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 781369eb-cd6b-42d1-98d4-6d4e477a37d3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64c4bcbb69-xgd8t namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 21fcd48e-105c-4d7a-be45-51cfdea3fb4d kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:01Z" name: model-chainer-64c4bcbb69-xgd8t.18a968c95440baf5 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 4e269d24-7cba-48d5-9086-9aa019d2d9aa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64c4bcbb69-xgd8t namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 21fcd48e-105c-4d7a-be45-51cfdea3fb4d kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:01Z" name: model-chainer-64c4bcbb69-xgd8t.18a968c955adb59a namespace: kserve-ci-e2e-test resourceVersion: "15389" uid: 3c96d2a1-e8f6-47d8-bee4-4b499bf0a590 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64c4bcbb69-xgd8t namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 21fcd48e-105c-4d7a-be45-51cfdea3fb4d kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-24T21:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:01Z" name: model-chainer-64c4bcbb69-xgd8t.18a968c958d66041 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: 84e4cd5c-ff25-4ad0-80ea-9662500a3a89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64c4bcbb69-xgd8t namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 21fcd48e-105c-4d7a-be45-51cfdea3fb4d kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-24T21:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:01Z" name: model-chainer-64c4bcbb69-xgd8t.18a968c959809533 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 7ebf0d35-f0e1-4239-933c-79e49cbe0160 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64c4bcbb69-xgd8t namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 21fcd48e-105c-4d7a-be45-51cfdea3fb4d kind: Event lastTimestamp: "2026-04-24T21:39:10Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-24T21:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:10Z" name: model-chainer-64c4bcbb69-xgd8t.18a968cb71aa7f3d namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: c4983d81-3b71-4838-8eca-be382ccc8c06 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64c4bcbb69-xgd8t namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 21fcd48e-105c-4d7a-be45-51cfdea3fb4d kind: Event lastTimestamp: "2026-04-24T21:39:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:38Z" name: model-chainer-64c4bcbb69-xgd8t.18a968cc047a965a namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: f23f5954-083c-48b8-b55c-8ad80ccb8cbb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-64c4bcbb69 namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: eec089b7-691a-42b1-9815-cc6b4a75adad kind: Event lastTimestamp: "2026-04-24T21:39:00Z" message: 'Created pod: model-chainer-64c4bcbb69-xgd8t' metadata: creationTimestamp: "2026-04-24T21:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:00Z" name: model-chainer-64c4bcbb69.18a968c916fc09e3 namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 68f8b3aa-d2d8-4424-9d30-278040e1bb32 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: 1d6b3fa8-cc3a-439b-9dbe-5779b950b4db kind: Event lastTimestamp: "2026-04-24T21:39:00Z" message: Scaled up replica set model-chainer-64c4bcbb69 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:00Z" name: model-chainer.18a968c91651bc74 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 33c4a232-81ca-4d95-9101-634a4fab124d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: 8d4498df-40d9-433a-aac7-8385d013c66f kind: Event lastTimestamp: "2026-04-24T21:39:00Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-24T21:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:00Z" name: model-chainer.18a968c919faaca3 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: d604f062-439b-4412-bfe5-76be00600f75 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:36.279993Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1ec3f-5f6c4cf864-t84z5 namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: 93c48cd8-ba73-462d-aaa2-e944e60e61dd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-1ec3f-5f6c4cf864-t84z5 to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:36Z" name: sequence-graph-1ec3f-5f6c4cf864-t84z5.18a9696b016236a6 namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: ac7927bf-9d56-44fe-ac8c-6b99f18183c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1ec3f-5f6c4cf864-t84z5 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 93c48cd8-ba73-462d-aaa2-e944e60e61dd kind: Event lastTimestamp: "2026-04-24T21:50:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-1ec3f-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:36Z" name: sequence-graph-1ec3f-5f6c4cf864-t84z5.18a9696b0d3b0f01 namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 8bfe945d-5141-4829-a184-8f776ce8a12e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1ec3f-5f6c4cf864-t84z5 namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 93c48cd8-ba73-462d-aaa2-e944e60e61dd kind: Event lastTimestamp: "2026-04-24T21:50:37Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:37Z" name: sequence-graph-1ec3f-5f6c4cf864-t84z5.18a9696b3eb24074 namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: df52d032-12bf-4b6d-a8ea-486b848de91a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1ec3f} kind: Pod name: sequence-graph-1ec3f-5f6c4cf864-t84z5 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 93c48cd8-ba73-462d-aaa2-e944e60e61dd kind: Event lastTimestamp: "2026-04-24T21:50:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:37Z" name: sequence-graph-1ec3f-5f6c4cf864-t84z5.18a9696b40056512 namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: a0d93b8c-4ee3-4e9f-be81-41a2a0697b02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1ec3f} kind: Pod name: sequence-graph-1ec3f-5f6c4cf864-t84z5 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 93c48cd8-ba73-462d-aaa2-e944e60e61dd kind: Event lastTimestamp: "2026-04-24T21:50:37Z" message: 'Created container: sequence-graph-1ec3f' metadata: creationTimestamp: "2026-04-24T21:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:37Z" name: sequence-graph-1ec3f-5f6c4cf864-t84z5.18a9696b431f7282 namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: 6408ed5a-d4ad-4c87-bad6-98537bcc3c38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1ec3f} kind: Pod name: sequence-graph-1ec3f-5f6c4cf864-t84z5 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 93c48cd8-ba73-462d-aaa2-e944e60e61dd kind: Event lastTimestamp: "2026-04-24T21:50:37Z" message: Started container sequence-graph-1ec3f metadata: creationTimestamp: "2026-04-24T21:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:37Z" name: sequence-graph-1ec3f-5f6c4cf864-t84z5.18a9696b439d70d7 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 319ccc8d-1fb1-477e-bd07-0610a00c7330 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1ec3f} kind: Pod name: sequence-graph-1ec3f-5f6c4cf864-t84z5 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 93c48cd8-ba73-462d-aaa2-e944e60e61dd kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: Stopping container sequence-graph-1ec3f metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:51Z" name: sequence-graph-1ec3f-5f6c4cf864-t84z5.18a969de31799b90 namespace: kserve-ci-e2e-test resourceVersion: "25050" uid: 4df0c931-9fd4-4f48-89fd-fd854c7c99d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:58:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1ec3f} kind: Pod name: sequence-graph-1ec3f-5f6c4cf864-t84z5 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 93c48cd8-ba73-462d-aaa2-e944e60e61dd kind: Event lastTimestamp: "2026-04-24T21:59:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:18Z" name: sequence-graph-1ec3f-5f6c4cf864-t84z5.18a969dec80e90f2 namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: fb0723ec-3d86-46c5-bde6-06b8d6f348c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-1ec3f-5f6c4cf864 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: 253580e0-e07d-4389-b654-253888e9115b kind: Event lastTimestamp: "2026-04-24T21:50:36Z" message: 'Created pod: sequence-graph-1ec3f-5f6c4cf864-t84z5' metadata: creationTimestamp: "2026-04-24T21:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:36Z" name: sequence-graph-1ec3f-5f6c4cf864.18a9696b00f2a56e namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: 86b48704-f94f-4c7f-9060-dc05ed36a121 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: e0af60f2-6af7-487e-acea-b4cc248efc2e kind: Event lastTimestamp: "2026-04-24T21:50:36Z" message: Scaled up replica set sequence-graph-1ec3f-5f6c4cf864 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:36Z" name: sequence-graph-1ec3f.18a9696b00570a96 namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 55d4e6a4-e3b6-45d9-9adb-4fa173d26930 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: d7dbfee2-b8c5-418c-8b05-84342bf9b53a kind: Event lastTimestamp: "2026-04-24T21:50:36Z" message: InferenceGraph [sequence-graph-1ec3f] is Ready metadata: creationTimestamp: "2026-04-24T21:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:36Z" name: sequence-graph-1ec3f.18a9696b05806233 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: 0af0bd01-a660-4267-91a8-db7fd0a38c01 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:50:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 217e06b3-670f-4fa4-8b76-d92665b0f669 kind: Event lastTimestamp: "2026-04-24T21:51:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:21Z" name: sequence-graph-1ec3f.18a9696e7ec0fcae namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: 38304d42-aa78-4783-9ea7-fa42be659860 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:50:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 217e06b3-670f-4fa4-8b76-d92665b0f669 kind: Event lastTimestamp: "2026-04-24T21:51:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:21Z" name: sequence-graph-1ec3f.18a9696e7efa53ba namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: 5c1ce043-da19-4b8d-9a09-ee2971770de8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:40:11.214985Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7fc99-59cc895d65-jhvth namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: 61786274-3c1c-4909-85cc-dc763bba1fe1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7fc99-59cc895d65-jhvth to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:40:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:40:11Z" name: sequence-graph-7fc99-59cc895d65-jhvth.18a968d9789b0f6e namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 7d392609-b8d2-469d-a5d5-4dc4603b7395 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7fc99-59cc895d65-jhvth namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 61786274-3c1c-4909-85cc-dc763bba1fe1 kind: Event lastTimestamp: "2026-04-24T21:40:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-7fc99-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:11Z" name: sequence-graph-7fc99-59cc895d65-jhvth.18a968d9899a4f1b namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: fd25540c-a718-4ae3-9fb8-1b4eb6e79ee1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:12Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7fc99-59cc895d65-jhvth namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 61786274-3c1c-4909-85cc-dc763bba1fe1 kind: Event lastTimestamp: "2026-04-24T21:40:12Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:40:12Z" name: sequence-graph-7fc99-59cc895d65-jhvth.18a968d9b56ea67e namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 450b4fd4-3b4e-4f08-980a-0465c3afaabe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7fc99} kind: Pod name: sequence-graph-7fc99-59cc895d65-jhvth namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 61786274-3c1c-4909-85cc-dc763bba1fe1 kind: Event lastTimestamp: "2026-04-24T21:40:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:12Z" name: sequence-graph-7fc99-59cc895d65-jhvth.18a968d9b6caccbc namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 941bea75-d49c-41ac-9219-c260a73e1184 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7fc99} kind: Pod name: sequence-graph-7fc99-59cc895d65-jhvth namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 61786274-3c1c-4909-85cc-dc763bba1fe1 kind: Event lastTimestamp: "2026-04-24T21:40:12Z" message: 'Created container: sequence-graph-7fc99' metadata: creationTimestamp: "2026-04-24T21:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:12Z" name: sequence-graph-7fc99-59cc895d65-jhvth.18a968d9ba675b08 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: c24cf6d2-d090-45b3-a780-814a5c3764c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7fc99} kind: Pod name: sequence-graph-7fc99-59cc895d65-jhvth namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 61786274-3c1c-4909-85cc-dc763bba1fe1 kind: Event lastTimestamp: "2026-04-24T21:40:12Z" message: Started container sequence-graph-7fc99 metadata: creationTimestamp: "2026-04-24T21:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:12Z" name: sequence-graph-7fc99-59cc895d65-jhvth.18a968d9badd0cf6 namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: e18e82d6-241e-4330-b46f-09ec59f38e5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7fc99} kind: Pod name: sequence-graph-7fc99-59cc895d65-jhvth namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 61786274-3c1c-4909-85cc-dc763bba1fe1 kind: Event lastTimestamp: "2026-04-24T21:48:25Z" message: Stopping container sequence-graph-7fc99 metadata: creationTimestamp: "2026-04-24T21:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:25Z" name: sequence-graph-7fc99-59cc895d65-jhvth.18a9694ca2e572f4 namespace: kserve-ci-e2e-test resourceVersion: "19785" uid: 47a7fdf8-4772-426e-be3b-bc3fb136a223 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:48:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7fc99} kind: Pod name: sequence-graph-7fc99-59cc895d65-jhvth namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 61786274-3c1c-4909-85cc-dc763bba1fe1 kind: Event lastTimestamp: "2026-04-24T21:48:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:53Z" name: sequence-graph-7fc99-59cc895d65-jhvth.18a9694d4a1210f6 namespace: kserve-ci-e2e-test resourceVersion: "20195" uid: e62c8a16-799a-4e41-90ab-36a5f1567516 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7fc99-59cc895d65 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: efda3f04-d08c-4b13-955b-695f7ab1dc73 kind: Event lastTimestamp: "2026-04-24T21:40:11Z" message: 'Created pod: sequence-graph-7fc99-59cc895d65-jhvth' metadata: creationTimestamp: "2026-04-24T21:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:11Z" name: sequence-graph-7fc99-59cc895d65.18a968d9781a61b2 namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: 0bcf6a4c-4492-49f7-8dda-f1f1fa2db7d5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7fc99 namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 7132a63c-e05f-4bc7-888b-b8a8010a7d53 kind: Event lastTimestamp: "2026-04-24T21:40:11Z" message: Scaled up replica set sequence-graph-7fc99-59cc895d65 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:11Z" name: sequence-graph-7fc99.18a968d9762e8a3d namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 3f6dbf7a-02c5-47fb-899b-f32e791239d6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7fc99 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 0fc733d4-2780-4825-80f2-31470b5a9f4d kind: Event lastTimestamp: "2026-04-24T21:40:11Z" message: InferenceGraph [sequence-graph-7fc99] is Ready metadata: creationTimestamp: "2026-04-24T21:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:11Z" name: sequence-graph-7fc99.18a968d97c3976c1 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 2fe637ec-5e1c-418f-b31d-8623e8ad9300 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7fc99 namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: 756a80d5-b949-443b-ac78-7fd4579d92cf kind: Event lastTimestamp: "2026-04-24T21:40:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:26Z" name: sequence-graph-7fc99.18a968dcf4ab1f4f namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 47186bd1-9bc2-4034-981d-47a6cb74e477 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7fc99 namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: 756a80d5-b949-443b-ac78-7fd4579d92cf kind: Event lastTimestamp: "2026-04-24T21:40:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:26Z" name: sequence-graph-7fc99.18a968dcf5114614 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: bdf27f95-de32-4ae9-92f3-f62d1c26f66e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:49:26.026436Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b9878-79b8776d89-hngl7 namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 95d06567-f7c3-4722-a71e-23e7c86ee03f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-b9878-79b8776d89-hngl7 to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878-79b8776d89-hngl7.18a9695aa5f1fabd namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: 5d0e3fb2-c4b3-4869-a5e1-3c9ff5adcf8b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b9878-79b8776d89-hngl7 namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 95d06567-f7c3-4722-a71e-23e7c86ee03f kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878-79b8776d89-hngl7.18a9695ac0267471 namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 1f639202-afdc-479c-b7a2-1466ac8d5a54 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b9878} kind: Pod name: sequence-graph-b9878-79b8776d89-hngl7 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 95d06567-f7c3-4722-a71e-23e7c86ee03f kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878-79b8776d89-hngl7.18a9695ac158f82b namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: dfcf4d7d-e9eb-4eaa-a8bf-ceb04b8a8db9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b9878} kind: Pod name: sequence-graph-b9878-79b8776d89-hngl7 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 95d06567-f7c3-4722-a71e-23e7c86ee03f kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'Created container: sequence-graph-b9878' metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878-79b8776d89-hngl7.18a9695ac47d8637 namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 85f8fdcd-95a9-4bef-b473-5971dbe384da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b9878} kind: Pod name: sequence-graph-b9878-79b8776d89-hngl7 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 95d06567-f7c3-4722-a71e-23e7c86ee03f kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: Started container sequence-graph-b9878 metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878-79b8776d89-hngl7.18a9695ac4f694a6 namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: 6f5013e6-0b6d-47b7-a454-00dcf351344f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b9878} kind: Pod name: sequence-graph-b9878-79b8776d89-hngl7 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 95d06567-f7c3-4722-a71e-23e7c86ee03f kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Stopping container sequence-graph-b9878 metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: sequence-graph-b9878-79b8776d89-hngl7.18a9695cfd1de13f namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: f334de69-a8d2-4051-95a4-fc573f22b33d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b9878} kind: Pod name: sequence-graph-b9878-79b8776d89-hngl7 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 95d06567-f7c3-4722-a71e-23e7c86ee03f kind: Event lastTimestamp: "2026-04-24T21:50:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:03Z" name: sequence-graph-b9878-79b8776d89-hngl7.18a9695d82e0498b namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 7c77ed72-ccf2-4a55-b59d-ed0bcc4b2172 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-b9878-79b8776d89 namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: 330f89b0-c8fa-4527-88ed-492ee649b421 kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'Created pod: sequence-graph-b9878-79b8776d89-hngl7' metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878-79b8776d89.18a9695aa5829b99 namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: 0ee2405c-7244-41ef-abe2-b062ab842b07 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-b9878 namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: d0194dc5-1a6e-4607-b8fd-eef8b91c34e9 kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: Scaled up replica set sequence-graph-b9878-79b8776d89 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878.18a9695aa4e55c56 namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: 56aa3237-767b-44d8-a63d-aa3c46b2ddc8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b9878 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: d31825a5-53eb-4b92-a05b-b1e1a6249e33 kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: InferenceGraph [sequence-graph-b9878] is Ready metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878.18a9695aaa291e63 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: 67275270-2539-4d98-ab87-03de6910572f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b9878 namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: d31825a5-53eb-4b92-a05b-b1e1a6249e33 kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'Failed to update status for InferenceGraph "sequence-graph-b9878": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-b9878": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878.18a9695aacc64711 namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 3d0e2819-1946-4f07-9c22-ad3260dec240 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b9878 namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: d31825a5-53eb-4b92-a05b-b1e1a6249e33 kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-b9878": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:26Z" name: sequence-graph-b9878.18a9695aacc69855 namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: cc4e82e5-063f-4f69-85df-a9bd42093932 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:00:31.655196Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-600f1-6878b7dd5c-cttln namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: dd12d41f-dd99-4536-b3e7-da6f0745ca15 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-600f1-6878b7dd5c-cttln to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T22:00:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:00:31Z" name: splitter-graph-600f1-6878b7dd5c-cttln.18a969f5a082ed28 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: a6fcaead-5393-4fe0-8b62-d253100092b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:32Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-600f1-6878b7dd5c-cttln namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: dd12d41f-dd99-4536-b3e7-da6f0745ca15 kind: Event lastTimestamp: "2026-04-24T22:00:32Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:00:32Z" name: splitter-graph-600f1-6878b7dd5c-cttln.18a969f5b98d47d8 namespace: kserve-ci-e2e-test resourceVersion: "26582" uid: 28466c3c-c213-45dd-824c-ced8dace7de0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-600f1} kind: Pod name: splitter-graph-600f1-6878b7dd5c-cttln namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: dd12d41f-dd99-4536-b3e7-da6f0745ca15 kind: Event lastTimestamp: "2026-04-24T22:00:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:32Z" name: splitter-graph-600f1-6878b7dd5c-cttln.18a969f5bafdd1c9 namespace: kserve-ci-e2e-test resourceVersion: "26584" uid: cd7a6da7-9ba3-4c46-8fa1-5e638acc28cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-600f1} kind: Pod name: splitter-graph-600f1-6878b7dd5c-cttln namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: dd12d41f-dd99-4536-b3e7-da6f0745ca15 kind: Event lastTimestamp: "2026-04-24T22:00:32Z" message: 'Created container: splitter-graph-600f1' metadata: creationTimestamp: "2026-04-24T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:32Z" name: splitter-graph-600f1-6878b7dd5c-cttln.18a969f5be7c510e namespace: kserve-ci-e2e-test resourceVersion: "26586" uid: 8cd74ae8-51da-43cf-be74-404f51bde29b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-600f1} kind: Pod name: splitter-graph-600f1-6878b7dd5c-cttln namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: dd12d41f-dd99-4536-b3e7-da6f0745ca15 kind: Event lastTimestamp: "2026-04-24T22:00:32Z" message: Started container splitter-graph-600f1 metadata: creationTimestamp: "2026-04-24T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:32Z" name: splitter-graph-600f1-6878b7dd5c-cttln.18a969f5bef5df71 namespace: kserve-ci-e2e-test resourceVersion: "26587" uid: 585d1203-f584-43cb-a0c9-6f6953ddb43f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-600f1} kind: Pod name: splitter-graph-600f1-6878b7dd5c-cttln namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: dd12d41f-dd99-4536-b3e7-da6f0745ca15 kind: Event lastTimestamp: "2026-04-24T22:08:46Z" message: Stopping container splitter-graph-600f1 metadata: creationTimestamp: "2026-04-24T22:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:46Z" name: splitter-graph-600f1-6878b7dd5c-cttln.18a96a68c969cf91 namespace: kserve-ci-e2e-test resourceVersion: "30018" uid: b7445778-e6a3-4ad4-90d3-f6902fbed3a7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-600f1} kind: Pod name: splitter-graph-600f1-6878b7dd5c-cttln namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: dd12d41f-dd99-4536-b3e7-da6f0745ca15 kind: Event lastTimestamp: "2026-04-24T22:09:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:09:13Z" name: splitter-graph-600f1-6878b7dd5c-cttln.18a96a694629ef9c namespace: kserve-ci-e2e-test resourceVersion: "30253" uid: 04e0f461-d902-48e5-902a-f2c905049872 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-600f1-6878b7dd5c namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: cbff543a-b950-4823-8068-0fc7f8c2ec41 kind: Event lastTimestamp: "2026-04-24T22:00:31Z" message: 'Created pod: splitter-graph-600f1-6878b7dd5c-cttln' metadata: creationTimestamp: "2026-04-24T22:00:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:31Z" name: splitter-graph-600f1-6878b7dd5c.18a969f5a014386b namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: 815ceefc-4b7f-4cb8-b5b8-54ed2ff725b4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-600f1 namespace: kserve-ci-e2e-test resourceVersion: "26552" uid: 7b2de323-edb8-4b1c-ace9-1f4dc0f1cc2e kind: Event lastTimestamp: "2026-04-24T22:00:31Z" message: Scaled up replica set splitter-graph-600f1-6878b7dd5c from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:00:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:31Z" name: splitter-graph-600f1.18a969f59f0610f5 namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: 5258927b-2c6c-43a6-aa4b-9bfa0fb1a31c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-600f1 namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: 71e2e811-7e10-4156-aa3d-6c9154720602 kind: Event lastTimestamp: "2026-04-24T22:00:31Z" message: InferenceGraph [splitter-graph-600f1] is Ready metadata: creationTimestamp: "2026-04-24T22:00:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:00:31Z" name: splitter-graph-600f1.18a969f5a39217f8 namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: fe8daee6-0ddd-49ef-ae3a-65854882309e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-600f1 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: c9edce44-b5b4-477d-9549-bbaf6ecc0477 kind: Event lastTimestamp: "2026-04-24T22:00:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:00:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:46Z" name: splitter-graph-600f1.18a969f91e113928 namespace: kserve-ci-e2e-test resourceVersion: "26703" uid: 7db8e94f-37fd-42f1-89ff-bb7c2fdfcce1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-600f1 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: c9edce44-b5b4-477d-9549-bbaf6ecc0477 kind: Event lastTimestamp: "2026-04-24T22:00:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:00:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:46Z" name: splitter-graph-600f1.18a969f91e8084a5 namespace: kserve-ci-e2e-test resourceVersion: "26704" uid: e8766847-9dd6-4572-acd0-703371368d10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:01:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-600f1 namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: c9edce44-b5b4-477d-9549-bbaf6ecc0477 kind: Event lastTimestamp: "2026-04-24T22:01:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:01:01Z" name: splitter-graph-600f1.18a969fc9d412a86 namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: 5f6b5c8f-27b8-49ee-83f7-0f131edf1b0f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:01:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-600f1 namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: c9edce44-b5b4-477d-9549-bbaf6ecc0477 kind: Event lastTimestamp: "2026-04-24T22:01:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:01:01Z" name: splitter-graph-600f1.18a969fc9db85e39 namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 1d429caf-a88d-444a-b105-6fc7d26908f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:59:21.385424Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a4170-65f94cfc4d-7lwc7 namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: 3d18794b-cdfc-4b07-9712-3c52c56b0a54 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a4170-65f94cfc4d-7lwc7 to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:59:21Z" name: splitter-graph-a4170-65f94cfc4d-7lwc7.18a969e5441b352e namespace: kserve-ci-e2e-test resourceVersion: "25721" uid: 4338cab2-a87c-4067-9b74-f4d70090758d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:21Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a4170-65f94cfc4d-7lwc7 namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: 3d18794b-cdfc-4b07-9712-3c52c56b0a54 kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:59:21Z" name: splitter-graph-a4170-65f94cfc4d-7lwc7.18a969e55d49e62b namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: 80edd07e-eea5-4528-9a47-cc870ea9a76b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a4170} kind: Pod name: splitter-graph-a4170-65f94cfc4d-7lwc7 namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: 3d18794b-cdfc-4b07-9712-3c52c56b0a54 kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:21Z" name: splitter-graph-a4170-65f94cfc4d-7lwc7.18a969e55ebb2bdf namespace: kserve-ci-e2e-test resourceVersion: "25741" uid: 0f6e6510-5b5b-45ed-a990-d5a3fde75c68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a4170} kind: Pod name: splitter-graph-a4170-65f94cfc4d-7lwc7 namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: 3d18794b-cdfc-4b07-9712-3c52c56b0a54 kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: 'Created container: splitter-graph-a4170' metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:21Z" name: splitter-graph-a4170-65f94cfc4d-7lwc7.18a969e561e0c8cf namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 53840204-d7b9-47ba-b03c-03d7c977b481 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a4170} kind: Pod name: splitter-graph-a4170-65f94cfc4d-7lwc7 namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: 3d18794b-cdfc-4b07-9712-3c52c56b0a54 kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: Started container splitter-graph-a4170 metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:21Z" name: splitter-graph-a4170-65f94cfc4d-7lwc7.18a969e562576e18 namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: fdf028d0-350d-44b9-9ddd-cd73b2122ed1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a4170} kind: Pod name: splitter-graph-a4170-65f94cfc4d-7lwc7 namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: 3d18794b-cdfc-4b07-9712-3c52c56b0a54 kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: Stopping container splitter-graph-a4170 metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:31Z" name: splitter-graph-a4170-65f94cfc4d-7lwc7.18a969e79b42c6d2 namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: 7352547c-7a8a-4bd7-ae72-d0b133a25e8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:59:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a4170} kind: Pod name: splitter-graph-a4170-65f94cfc4d-7lwc7 namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: 3d18794b-cdfc-4b07-9712-3c52c56b0a54 kind: Event lastTimestamp: "2026-04-24T21:59:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:58Z" name: splitter-graph-a4170-65f94cfc4d-7lwc7.18a969e8006fe4cc namespace: kserve-ci-e2e-test resourceVersion: "26271" uid: fc90cfb1-52c4-439e-847e-b8e08bd36f4f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a4170-65f94cfc4d namespace: kserve-ci-e2e-test resourceVersion: "25710" uid: 6d7d5e86-a041-4d52-b274-02d28c3f4d1d kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: 'Created pod: splitter-graph-a4170-65f94cfc4d-7lwc7' metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:21Z" name: splitter-graph-a4170-65f94cfc4d.18a969e5439316c7 namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 7fa97360-b561-4b2b-86b4-1d80aa4b4bb6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a4170 namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: 607c46b4-4532-4318-9357-a3fa34f048d2 kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: Scaled up replica set splitter-graph-a4170-65f94cfc4d from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:21Z" name: splitter-graph-a4170.18a969e541b94c53 namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: e4147b7f-e46f-4fb4-ad8b-52d545ecd137 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a4170 namespace: kserve-ci-e2e-test resourceVersion: "25731" uid: 8544eee9-42f1-47e9-98a4-5838ac8bae3b kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: InferenceGraph [splitter-graph-a4170] is Ready metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:21Z" name: splitter-graph-a4170.18a969e5492bc504 namespace: kserve-ci-e2e-test resourceVersion: "25733" uid: 18e0ad22-d814-4e40-a20f-242a78aee8c5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:48:56.516961Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1dabb-predictor-68d6866787-m6hgq to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a96953c70b20bb namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 2c106211-2451-4c7f-8793-ee74939c59f9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1dabb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a96953d46842e4 namespace: kserve-ci-e2e-test resourceVersion: "20333" uid: 2c8c9eb5-47c9-402d-8cad-2487294edff6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:48:57Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a969540485299f namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 87eeb14f-8de7-4cfb-bfeb-d675a15eda43 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a96954064901a0 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: a2d4d351-4f98-408a-a8db-78e4f3ac2f9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a969540a01823a namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 4f0e2b1b-fa49-4bc4-9db5-1ba279ba3acc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a969540a71b0d3 namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: a8f79114-ee6c-47d1-ac5d-a0e62fe80823 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a969540a7df8ef namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: 5a152c99-c318-49d1-beef-7e1b3df2685a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a969540e404823 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2e798479-182d-4406-9113-cdf933da04f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:48:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:57Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a969540eb31019 namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 5e0f3c0e-b4a8-4f09-af26-fc6f08f91910 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:48:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:49:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:44Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a96954571f627a namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: 85a76e3f-fb36-4ed2-a730-cf8e256bda2b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a969d742a7dd02 namespace: kserve-ci-e2e-test resourceVersion: "24707" uid: fdeb138b-f444-43a5-ac74-42a5add1a7f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: eda051e5-7bbf-41a9-ad1b-c9d1e5368798 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: success-200-isvc-1dabb-predictor-68d6866787-m6hgq.18a969d742a84a13 namespace: kserve-ci-e2e-test resourceVersion: "24710" uid: 43080d30-7bab-452c-94cb-2d9a6ec15dfe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1dabb-predictor-68d6866787 namespace: kserve-ci-e2e-test resourceVersion: "20271" uid: 64a016d4-4ecd-47dd-8193-c28201ada65f kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'Created pod: success-200-isvc-1dabb-predictor-68d6866787-m6hgq' metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-1dabb-predictor-68d6866787.18a96953c6854ff7 namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: 2a73791d-1f4a-49d9-948a-18203f2db3a6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: d9a52153-1e70-459a-86d4-71bfdc2fe993 kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: Scaled up replica set success-200-isvc-1dabb-predictor-68d6866787 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-1dabb-predictor.18a96953c4b89fdd namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: 7f28975a-454f-47af-9d52-c17cabc77259 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 69626af7-3e17-4b45-b8a7-e51eda432813 kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: success-200-isvc-1dabb-predictor.18a96957432bdbf1 namespace: kserve-ci-e2e-test resourceVersion: "20665" uid: 08c54b95-3f92-4022-8172-9dad55244321 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:49:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 69626af7-3e17-4b45-b8a7-e51eda432813 kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: success-200-isvc-1dabb-predictor.18a9695743933695 namespace: kserve-ci-e2e-test resourceVersion: "20668" uid: a2798f3d-ae1c-4da9-8250-a26106150453 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 69626af7-3e17-4b45-b8a7-e51eda432813 kind: Event lastTimestamp: "2026-04-24T21:50:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:11Z" name: success-200-isvc-1dabb-predictor.18a9695e4170a94c namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: e4850f67-4ea5-4a95-bbfc-7f8e52d060ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1dabb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 69626af7-3e17-4b45-b8a7-e51eda432813 kind: Event lastTimestamp: "2026-04-24T21:50:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:11Z" name: success-200-isvc-1dabb-predictor.18a9695e41bb9272 namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 0ac254fb-e4e5-4fcb-b462-da195b030a0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1dabb namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: c2231702-f8c4-4fde-a355-53b8ee2d71a0 kind: Event lastTimestamp: "2026-04-24T21:49:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:55Z" name: success-200-isvc-1dabb.18a96953c48fe36e namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 6bae8582-4a07-4cbb-8f03-17c9efd0ce65 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1dabb namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: c2231702-f8c4-4fde-a355-53b8ee2d71a0 kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-1dabb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1dabb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-1dabb.18a96953c4ffc59e namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: fda1a849-9418-46e4-a571-9bd099d1dedc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1dabb namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: c2231702-f8c4-4fde-a355-53b8ee2d71a0 kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1dabb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-1dabb.18a96953c5003657 namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: 1792f0d2-06cb-430f-85a2-581b833cd4c9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1dabb namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: c2231702-f8c4-4fde-a355-53b8ee2d71a0 kind: Event lastTimestamp: "2026-04-24T21:49:55Z" message: InferenceService [success-200-isvc-1dabb] is Ready metadata: creationTimestamp: "2026-04-24T21:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:55Z" name: success-200-isvc-1dabb.18a969616917ea92 namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: 88e111a7-ecc1-49ee-8629-56f4fca632de reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:49:36.232687Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20807" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1ec3f-predictor-677b5997f5-brmms to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a9695d064908ba namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: 71b4c7d6-438f-4572-971d-5ae80d70cae8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a9695d1fe8b2ab namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: b3d81d7e-8a7d-4e19-a5f4-664e42faf51a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a9695d21df74ed namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 162fb215-2193-42ff-831e-4d991c58e232 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a9695d250afc5b namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: a68819bc-2d15-42ea-b66a-7a9cd4464c7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a9695d257d2f55 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 7a460b89-2e45-4fb6-8ddb-4e42847bbe05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a9695d258a79fa namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 35eb342f-7c41-43ac-ba14-a69bda32c06c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a9695d296196d4 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 4b493979-b4f3-4362-be38-a24c42d088d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a9695d29d36992 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 499fe4d6-0a8e-4b46-9e4c-a82956c25bad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:58:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:54Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a9695d74d1dc3c namespace: kserve-ci-e2e-test resourceVersion: "25203" uid: 7f45dc58-9c24-4f91-8a6a-ec99888d0e00 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a969de37d06b5a namespace: kserve-ci-e2e-test resourceVersion: "25072" uid: 33982c06-a79c-4dd3-a284-3dec68eb57d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a969de37d0e70e namespace: kserve-ci-e2e-test resourceVersion: "25076" uid: a63d7e97-91cf-4829-8d34-36ec9bbf6117 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 59803781-86f5-4c54-91c2-18922795d983 kind: Event lastTimestamp: "2026-04-24T21:58:54Z" message: 'Readiness probe failed: Get "https://10.134.0.48:8643/healthz": dial tcp 10.134.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:54Z" name: success-200-isvc-1ec3f-predictor-677b5997f5-brmms.18a969dee94390e4 namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 0bff064b-aa99-489b-a0a3-4555f3ca4055 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1ec3f-predictor-677b5997f5 namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: fee9e605-31bc-49c0-807b-8e1d311b26db kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: 'Created pod: success-200-isvc-1ec3f-predictor-677b5997f5-brmms' metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor-677b5997f5.18a9695d05b84008 namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: d9b675c8-1989-45f3-af1a-1737a22c7a73 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: 6819e788-063d-4fd9-bc3f-ca5c762f137a kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Scaled up replica set success-200-isvc-1ec3f-predictor-677b5997f5 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-1ec3f-predictor.18a9695d03a5e759 namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 408ecd35-49f3-4ca8-b55e-0512de66b5f0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: fd8114b3-01ad-4550-a2f1-81ce17114c49 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:21Z" name: success-200-isvc-1ec3f-predictor.18a9696082248fca namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: d31604dd-25a5-4503-9151-fa98f18d1977 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: fd8114b3-01ad-4550-a2f1-81ce17114c49 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:21Z" name: success-200-isvc-1ec3f-predictor.18a96960828f43ab namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: 458225e2-b93b-4337-8b35-0f9f49223e3e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: fd8114b3-01ad-4550-a2f1-81ce17114c49 kind: Event lastTimestamp: "2026-04-24T21:51:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:06Z" name: success-200-isvc-1ec3f-predictor.18a9696afef346e0 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 7a178029-66c9-43ac-a4f3-37c43eff6158 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ec3f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: fd8114b3-01ad-4550-a2f1-81ce17114c49 kind: Event lastTimestamp: "2026-04-24T21:51:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:06Z" name: success-200-isvc-1ec3f-predictor.18a9696aff52a2e1 namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 6add9aea-b4bd-4063-861a-47abab065b93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: 70f1dadf-18bb-4a84-9191-ad1a40b18b6d kind: Event lastTimestamp: "2026-04-24T21:50:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:24Z" name: success-200-isvc-1ec3f.18a9695d03711d1d namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: b31aa3c1-d1cc-43b5-97f9-8dff381624d0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ec3f namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: 70f1dadf-18bb-4a84-9191-ad1a40b18b6d kind: Event lastTimestamp: "2026-04-24T21:50:24Z" message: InferenceService [success-200-isvc-1ec3f] is Ready metadata: creationTimestamp: "2026-04-24T21:50:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:24Z" name: success-200-isvc-1ec3f.18a969682fd71a6f namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 8b2a12f9-63d7-4d5e-ac91-9976e92fbc74 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:47:46.164989Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19387" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2525b-predictor-67d9995cb7-lcmhq to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:47:46Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a9694365bd34a0 namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: c10a83f1-39b3-4c62-9863-eba298c18ab8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2525b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:46Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a969436ed8b913 namespace: kserve-ci-e2e-test resourceVersion: "19427" uid: cf648d90-524c-4b64-bc7f-f90485051854 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:47:47Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96943a32d1458 namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: 91eb89a0-f479-4261-8a0d-785ca9685d83 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96943a4beb50a namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: ee4734e5-0d21-4056-b053-92619a1192a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96943a95e5038 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 6e434d63-f1d4-4943-a661-16c4ab67403d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96943a9c8c836 namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: 885c5f98-813f-4890-b974-5ef55c7e8f6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96943aa0154ea namespace: kserve-ci-e2e-test resourceVersion: "19452" uid: 737c3b53-009e-4593-9d09-76494ac7be09 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96943ac890531 namespace: kserve-ci-e2e-test resourceVersion: "19454" uid: 44996cae-a413-4d45-a4d1-4e221d42509a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:47:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:47Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96943acfb350c namespace: kserve-ci-e2e-test resourceVersion: "19456" uid: 86d4cbb7-1f11-4383-ab43-cca8313619a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:48:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:24Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96943fa48c6b5 namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: 88f7be68-8ac2-44fd-8de7-2a89b1f278ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96953c3e8c4bd namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 93e1ca4b-ac5e-4df3-82b9-4fa0eff380c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a96953c3e9322c namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: 624c22ee-da95-4c3a-a316-dbee4cc506e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 550706c4-48b4-456b-bd40-364ec1b3d86d kind: Event lastTimestamp: "2026-04-24T21:48:59Z" message: 'Readiness probe failed: Get "https://10.134.0.42:8643/healthz": dial tcp 10.134.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:59Z" name: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq.18a969548290c7cc namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: c8626204-0fc8-4424-8faf-5af8134a068a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2525b-predictor-67d9995cb7 namespace: kserve-ci-e2e-test resourceVersion: "19364" uid: a41a6f02-eb01-4530-9a09-d2a21e5c6505 kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: 'Created pod: success-200-isvc-2525b-predictor-67d9995cb7-lcmhq' metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:46Z" name: success-200-isvc-2525b-predictor-67d9995cb7.18a969436539d4e4 namespace: kserve-ci-e2e-test resourceVersion: "19389" uid: 55c034fa-93c5-4e57-a731-83c127d9cbea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: ad3fa144-09fd-403e-972a-536e3194fe36 kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: Scaled up replica set success-200-isvc-2525b-predictor-67d9995cb7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:47:46Z" name: success-200-isvc-2525b-predictor.18a9694361430dea namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: 5add3cff-11dc-48a3-a1ba-e0bc38f22306 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 7115927e-c86d-4b61-9fa8-681998057a8f kind: Event lastTimestamp: "2026-04-24T21:48:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:01Z" name: success-200-isvc-2525b-predictor.18a96946dfd05552 namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: d508dfde-124f-4a7d-adf6-59b3aebafa9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 7115927e-c86d-4b61-9fa8-681998057a8f kind: Event lastTimestamp: "2026-04-24T21:48:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:01Z" name: success-200-isvc-2525b-predictor.18a96946e037424f namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: b8ea10cb-6929-4bb3-8c65-35a7f1534c09 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:48:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 7115927e-c86d-4b61-9fa8-681998057a8f kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:46Z" name: success-200-isvc-2525b-predictor.18a9694a5ef88793 namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: 096a4d64-100c-4c7b-bb5c-d9ce1863c80a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:48:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2525b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 7115927e-c86d-4b61-9fa8-681998057a8f kind: Event lastTimestamp: "2026-04-24T21:48:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:46Z" name: success-200-isvc-2525b-predictor.18a9694a5f3fd6a4 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: ef9dbe0f-9c97-404f-a49b-d031b29d0591 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2525b namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 51e66bcb-3b21-4106-b9e1-f4d788bf1e0c kind: Event lastTimestamp: "2026-04-24T21:48:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:34Z" name: success-200-isvc-2525b.18a96943613e0754 namespace: kserve-ci-e2e-test resourceVersion: "20011" uid: fe95e1c0-25ec-40d0-aeed-40169f061bfa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2525b namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 51e66bcb-3b21-4106-b9e1-f4d788bf1e0c kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: 'Failed to update status for InferenceService "success-200-isvc-2525b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2525b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:46Z" name: success-200-isvc-2525b.18a9694361e9170f namespace: kserve-ci-e2e-test resourceVersion: "19370" uid: b67ede15-6c41-4182-a7b5-fe5467e34096 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2525b namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 51e66bcb-3b21-4106-b9e1-f4d788bf1e0c kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2525b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:47:46Z" name: success-200-isvc-2525b.18a9694361e9780a namespace: kserve-ci-e2e-test resourceVersion: "19371" uid: ae13cd6d-8631-4ccd-830a-863e7b9efaa7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2525b namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: 51e66bcb-3b21-4106-b9e1-f4d788bf1e0c kind: Event lastTimestamp: "2026-04-24T21:48:34Z" message: InferenceService [success-200-isvc-2525b] is Ready metadata: creationTimestamp: "2026-04-24T21:48:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:34Z" name: success-200-isvc-2525b.18a9694eb7b2d5cf namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: 71a5981a-9c52-477b-8f26-0058447b9809 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:59:31.630145Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:59:31Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e7a6bd34df namespace: kserve-ci-e2e-test resourceVersion: "25886" uid: 667ab09d-afe2-48fb-a261-16a10fd4af93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-600f1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:31Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e7b2c5190f namespace: kserve-ci-e2e-test resourceVersion: "25925" uid: b5d4595d-b6c0-4d8a-8f6b-3cc68c35ba7d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Add eth0 [10.134.0.54/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:59:32Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e7e3f27dc8 namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: 0aae1b79-84c9-412b-9f11-40cf715aca08 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e7e5877167 namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: 92f19af6-54a9-4c73-ae0f-e617632e62b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e7e94857b4 namespace: kserve-ci-e2e-test resourceVersion: "25957" uid: 702282c2-7815-45d9-9451-3bbd0aa1959f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e7e9bb467e namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 44e5897f-d842-4cea-b098-8914515a1b56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e7e9c6cc75 namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: 64e8cf5b-7b11-45f8-8994-9b9dfd48b441 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e7ecfb8146 namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: 7524d4c6-7212-41c9-b235-92fb3bef51be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T21:59:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:32Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e7ed7022f1 namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 0bf08b05-3f8e-4017-8db1-a6d6a70f5390 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:59:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T22:08:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.54:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:49Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a969e8044a5df4 namespace: kserve-ci-e2e-test resourceVersion: "30072" uid: dd07be45-fa57-409b-8955-7d1d29907c39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T22:08:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:46Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a96a68ceed0b48 namespace: kserve-ci-e2e-test resourceVersion: "30032" uid: d0dc7688-f605-435d-b055-50ac6a0ad743 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:08:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T22:08:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:46Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a96a68ceed7633 namespace: kserve-ci-e2e-test resourceVersion: "30034" uid: 5c5fa21e-9045-4618-a045-f0cc5e31c746 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: 26fbe127-e1af-4d43-a766-a61d05cee245 kind: Event lastTimestamp: "2026-04-24T22:08:49Z" message: 'Readiness probe failed: Get "https://10.134.0.54:8643/healthz": dial tcp 10.134.0.54:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:49Z" name: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv.18a96a6978b80156 namespace: kserve-ci-e2e-test resourceVersion: "30070" uid: 2aa61509-5663-41a7-8701-d5ad728cbed1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-600f1-predictor-6d4bfcf845 namespace: kserve-ci-e2e-test resourceVersion: "25871" uid: ccf91b64-0cc9-46c3-a060-4a3c2a01f9f8 kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: 'Created pod: success-200-isvc-600f1-predictor-6d4bfcf845-sdtdv' metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:31Z" name: success-200-isvc-600f1-predictor-6d4bfcf845.18a969e7a5e94c6c namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: f40900e7-3f43-4206-b2a3-2033b10c38cf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25867" uid: e223b207-1abc-47a6-9ff0-16810c3512a3 kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: Scaled up replica set success-200-isvc-600f1-predictor-6d4bfcf845 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:31Z" name: success-200-isvc-600f1-predictor.18a969e7a2fa7844 namespace: kserve-ci-e2e-test resourceVersion: "25873" uid: 6273def0-9598-4d9f-8bd1-f19ba51bc88a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:59:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25869" uid: 93408058-a780-432d-be1e-9645dd3480da kind: Event lastTimestamp: "2026-04-24T22:00:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:59:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:01Z" name: success-200-isvc-600f1-predictor.18a969eb21e9a3f0 namespace: kserve-ci-e2e-test resourceVersion: "26299" uid: f5026caf-1bbc-4933-8b3c-f9da5f5a774f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:59:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25869" uid: 93408058-a780-432d-be1e-9645dd3480da kind: Event lastTimestamp: "2026-04-24T22:00:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:59:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:01Z" name: success-200-isvc-600f1-predictor.18a969eb225a278d namespace: kserve-ci-e2e-test resourceVersion: "26300" uid: 7d045fb5-d64c-4b1c-a17b-1b9352397e47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:00:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26122" uid: 93408058-a780-432d-be1e-9645dd3480da kind: Event lastTimestamp: "2026-04-24T22:00:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:46Z" name: success-200-isvc-600f1-predictor.18a969f22080ca92 namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: e3ba4cc3-c91c-4414-9e93-b75443ccb0ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:00:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-600f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26122" uid: 93408058-a780-432d-be1e-9645dd3480da kind: Event lastTimestamp: "2026-04-24T22:00:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:46Z" name: success-200-isvc-600f1-predictor.18a969f220cc647c namespace: kserve-ci-e2e-test resourceVersion: "26706" uid: 631bf93c-e638-4c11-9f33-c65d52aedad3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-600f1 namespace: kserve-ci-e2e-test resourceVersion: "25847" uid: 8241ced3-e9dc-4d96-abd8-73e12b84ce1a kind: Event lastTimestamp: "2026-04-24T22:00:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:00:19Z" name: success-200-isvc-600f1.18a969e7a2b223b6 namespace: kserve-ci-e2e-test resourceVersion: "26459" uid: 0d643809-4da2-40cd-ad2b-141a865524b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-600f1 namespace: kserve-ci-e2e-test resourceVersion: "25847" uid: 8241ced3-e9dc-4d96-abd8-73e12b84ce1a kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-600f1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-600f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:31Z" name: success-200-isvc-600f1.18a969e7a3403db8 namespace: kserve-ci-e2e-test resourceVersion: "25876" uid: 5c99e342-6f92-46cc-917d-476b83e2ee70 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-600f1 namespace: kserve-ci-e2e-test resourceVersion: "25847" uid: 8241ced3-e9dc-4d96-abd8-73e12b84ce1a kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-600f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:31Z" name: success-200-isvc-600f1.18a969e7a34093c0 namespace: kserve-ci-e2e-test resourceVersion: "25878" uid: 2a0a6320-e9d6-45de-b41f-8c57fadf4d4f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-600f1 namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 8241ced3-e9dc-4d96-abd8-73e12b84ce1a kind: Event lastTimestamp: "2026-04-24T22:00:19Z" message: InferenceService [success-200-isvc-600f1] is Ready metadata: creationTimestamp: "2026-04-24T22:00:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:00:19Z" name: success-200-isvc-600f1.18a969f2c0e1939c namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: 2a9eb939-22b3-4c78-a3aa-3f75c8b968b7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:11.142093Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a968cb7bfb8f74 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: bfc19b96-df7e-4f87-83bb-c0c1488cfdba reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a968cb95ba9069 namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: ab0785ac-d8af-4d44-b812-254cdceaa229 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a968cb96ff66ea namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 25a2e615-9e1f-40ac-95e4-0178d8128a49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a968cb9a343724 namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: d438831b-7967-4e50-9983-4830949677a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a968cb9aa39de4 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 1a6e49ae-ce29-4c8e-b4e3-76f83d05173a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a968cb9ab11e96 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 67257c2f-e7dd-4f3f-b7e8-a655de7d9096 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a968cb9d7374cc namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 2c3f3797-5e73-41f5-97a0-d44a5c7f2f6d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a968cb9ded5d94 namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 0daeed1a-5313-45a9-b978-a38fde7edda1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:48:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:27Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a968cba7ca497f namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 22777389-1cd4-45ed-a4b6-9a0b116708ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:48:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:25Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a9694caa930531 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 9aeae285-016a-41a0-935f-f4b665ae00a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:48:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:25Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a9694caa939287 namespace: kserve-ci-e2e-test resourceVersion: "19817" uid: 38d50541-812a-4317-a5b1-856847fd91c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 6a382bf3-3608-40fd-ab84-8604e941a8c3 kind: Event lastTimestamp: "2026-04-24T21:48:27Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:27Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6.18a9694d1c5d067f namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: ba8f528b-542a-45ec-b1b8-0ddef0a17078 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7fc99-predictor-5fbf44ffcf namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: fb4f340f-55f5-4aaa-8db0-3be06a9f23df kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: 'Created pod: success-200-isvc-7fc99-predictor-5fbf44ffcf-rllc6' metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor-5fbf44ffcf.18a968cb7b681ac7 namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: bdb1eaa3-9a26-4995-97a7-c25ef8fd86f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7fc99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 180b6bef-f666-45ad-a6fe-e6995e8bc969 kind: Event lastTimestamp: "2026-04-24T21:39:11Z" message: Scaled up replica set success-200-isvc-7fc99-predictor-5fbf44ffcf from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:11Z" name: success-200-isvc-7fc99-predictor.18a968cb796aeb33 namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 9dc55ec9-81c1-493a-974f-1f78d0c0db4e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7fc99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: b69bdc0d-317a-44dc-9b60-863f950955bb kind: Event lastTimestamp: "2026-04-24T21:40:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:26Z" name: success-200-isvc-7fc99-predictor.18a968cef7eb4e84 namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 4d05af48-8c54-4837-b2df-9547691f50db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7fc99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: b69bdc0d-317a-44dc-9b60-863f950955bb kind: Event lastTimestamp: "2026-04-24T21:40:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:26Z" name: success-200-isvc-7fc99-predictor.18a968cef8688367 namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: fa85ae63-a253-4676-835b-0782ad5162af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7fc99 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: 7f39f352-665e-4f93-82ad-bcf3ca371aea kind: Event lastTimestamp: "2026-04-24T21:39:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:58Z" name: success-200-isvc-7fc99.18a968cb794b6f73 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 5aec44dc-cbfe-4b63-8293-2dc7ed516686 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7fc99 namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 7f39f352-665e-4f93-82ad-bcf3ca371aea kind: Event lastTimestamp: "2026-04-24T21:39:57Z" message: InferenceService [success-200-isvc-7fc99] is Ready metadata: creationTimestamp: "2026-04-24T21:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:57Z" name: success-200-isvc-7fc99.18a968d663484ef0 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 7d8899a7-5bf6-4d38-aff8-fa41dc749ad9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:37:01.103395Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-914c2-predictor-7db85f97bf-5zqtz to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:37:01Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968ad35127da4 namespace: kserve-ci-e2e-test resourceVersion: "13944" uid: 4f8993f5-1646-414c-bf93-16b5d431a640 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-914c2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:01Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968ad3d413350 namespace: kserve-ci-e2e-test resourceVersion: "13976" uid: 998017b4-1966-4672-b970-8b346b21d2bc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:02Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:37:02Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968ad7eba0d76 namespace: kserve-ci-e2e-test resourceVersion: "14037" uid: c956bad5-9974-43aa-ba0b-8b792e54facc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:02Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" metadata: creationTimestamp: "2026-04-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:02Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968ad80107f52 namespace: kserve-ci-e2e-test resourceVersion: "14039" uid: 4ef3dbe4-1c16-4416-908b-d8f8405ab734 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:18Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" in 15.847s (15.847s including waiting). Image size: 1335702126 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:18Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968b130a131b4 namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: 992d3223-27d1-4f39-b8de-2a643acbc3ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:18Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968b134aef7dd namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: b4d58860-d8c8-49e6-922d-d509cd413df5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:18Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968b135249f28 namespace: kserve-ci-e2e-test resourceVersion: "14239" uid: 6c052bfd-fef6-40df-be71-867c08fe99e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:18Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968b135310b06 namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: e286c6b2-0cec-45c3-b2a3-3053a85dfb0b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:20Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.643s (2.643s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:20Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968b1d2be0e29 namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: c0b854ef-0ea1-42c3-9729-7d2818ac7bd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:21Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968b1d7b5a443 namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: d98a4a0b-4f42-46a9-b76c-9b777d4c365e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:21Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968b1d86ed008 namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: 0d8a9038-e9db-4167-868f-53f1a0f360b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:37:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:37:58Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968b22fff8c48 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: ab679ed3-d533-4c4b-bfcd-13a6ebd0525d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968c22e17620f namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: 2e007da2-8728-4557-8251-ba7f17a3efc9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968c22e17e0d3 namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 75f083aa-91c3-40bc-924f-cdc7d3a6727c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 49d0a4d9-3576-4587-ab0b-e2df7f7b45c8 kind: Event lastTimestamp: "2026-04-24T21:38:33Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:33Z" name: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz.18a968c2b8153773 namespace: kserve-ci-e2e-test resourceVersion: "15082" uid: ba282b82-c2f0-4992-82c6-c3f5d1b025b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-914c2-predictor-7db85f97bf namespace: kserve-ci-e2e-test resourceVersion: "13917" uid: b12996af-d0f2-4cd2-b0de-445e86a7a3d8 kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: 'Created pod: success-200-isvc-914c2-predictor-7db85f97bf-5zqtz' metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:01Z" name: success-200-isvc-914c2-predictor-7db85f97bf.18a968ad34833de6 namespace: kserve-ci-e2e-test resourceVersion: "13941" uid: 128678ba-bb7e-41c3-b7da-a49be075e658 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13915" uid: fbc05153-67f2-4ca0-8e15-5d38759ca4bc kind: Event lastTimestamp: "2026-04-24T21:37:01Z" message: Scaled up replica set success-200-isvc-914c2-predictor-7db85f97bf from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:01Z" name: success-200-isvc-914c2-predictor.18a968ad2f8dee16 namespace: kserve-ci-e2e-test resourceVersion: "13920" uid: f9a4dab1-75f2-44d2-9a97-238cdffe729a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13916" uid: 4e9d4eaa-1a40-419c-a478-bd13ce60f425 kind: Event lastTimestamp: "2026-04-24T21:37:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:46Z" name: success-200-isvc-914c2-predictor.18a968b0ae17ca68 namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 93283083-9f83-4542-9452-95b3dd703bd6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:37:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13916" uid: 4e9d4eaa-1a40-419c-a478-bd13ce60f425 kind: Event lastTimestamp: "2026-04-24T21:37:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:37:46Z" name: success-200-isvc-914c2-predictor.18a968b0ae519398 namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: 6a18a48d-f5ff-4686-8c16-3e94568692f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: 4e9d4eaa-1a40-419c-a478-bd13ce60f425 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-914c2-predictor.18a968bb2ab97eb5 namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: 6752957f-adaf-4147-b2e0-fdd795154767 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:38:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-914c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: 4e9d4eaa-1a40-419c-a478-bd13ce60f425 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-914c2-predictor.18a968bb2b1c9681 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: edd42eac-484a-4650-a444-42afabb2c61a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:37:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-914c2 namespace: kserve-ci-e2e-test resourceVersion: "13906" uid: a9e7a1c6-4719-4a5e-b81c-e29e8b268e0c kind: Event lastTimestamp: "2026-04-24T21:38:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:08Z" name: success-200-isvc-914c2.18a968ad2f683bbb namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 8bf1d226-ef57-48a6-82d0-cfbec6c49f25 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-914c2 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: a9e7a1c6-4719-4a5e-b81c-e29e8b268e0c kind: Event lastTimestamp: "2026-04-24T21:38:08Z" message: InferenceService [success-200-isvc-914c2] is Ready metadata: creationTimestamp: "2026-04-24T21:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:08Z" name: success-200-isvc-914c2.18a968bcee57f3b9 namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: 9d4b2880-b352-4fb9-8613-11e4aa54c3ac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:58:21.328362Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24734" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a4170-predictor-c76b89db5-hv7sv to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:58:21Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d7486d2ee9 namespace: kserve-ci-e2e-test resourceVersion: "24743" uid: 24360a93-711c-4840-a0d6-356970bb11d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a4170-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:21Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d7531e2db5 namespace: kserve-ci-e2e-test resourceVersion: "24772" uid: c730848b-6097-4cd9-a6a6-72962b2c1f6a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24755" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:58:22Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:58:22Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d785cc25b4 namespace: kserve-ci-e2e-test resourceVersion: "24794" uid: 8c99ec92-a05d-4cba-b7b0-da993fcfb3f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:58:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:22Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d78757634d namespace: kserve-ci-e2e-test resourceVersion: "24798" uid: b03e8ad2-3bfb-4c28-9830-41ad4165c560 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:58:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:22Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d78a8352f9 namespace: kserve-ci-e2e-test resourceVersion: "24799" uid: 81b63a87-669a-4129-9d9f-aa66d73112ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:58:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:22Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d78afa9101 namespace: kserve-ci-e2e-test resourceVersion: "24800" uid: 65a2b394-27f3-4109-8100-0aeace37f66f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:58:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:22Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d78b0d10d7 namespace: kserve-ci-e2e-test resourceVersion: "24801" uid: a40b4b3c-3876-49e8-b3a7-bee4316b57e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:58:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:22Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d78db60c47 namespace: kserve-ci-e2e-test resourceVersion: "24802" uid: 86462c33-befe-4f2b-abbf-1667835b7da7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:58:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:22Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d78e243a22 namespace: kserve-ci-e2e-test resourceVersion: "24803" uid: b2d1d213-618d-4c8f-aa67-b24ab7bd7720 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:58:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:58:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:59Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969d7e200f3a2 namespace: kserve-ci-e2e-test resourceVersion: "25332" uid: 4818cb00-58f1-4c4a-8bc4-d35489ce6eac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:31Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969e7a32181ab namespace: kserve-ci-e2e-test resourceVersion: "25875" uid: 057b2c04-abd0-4eaa-ba7b-65652564be70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:59:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:31Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969e7a32229df namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: da2bc8eb-a2d4-45f0-9848-e6ea84fcca2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: 8757ba4b-dc3a-4ef0-9c03-f465879a9ad7 kind: Event lastTimestamp: "2026-04-24T21:59:34Z" message: 'Readiness probe failed: Get "https://10.134.0.50:8643/healthz": dial tcp 10.134.0.50:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:34Z" name: success-200-isvc-a4170-predictor-c76b89db5-hv7sv.18a969e86a5881d6 namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: 457f1955-30ca-4fc5-8672-cd5386cccb5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a4170-predictor-c76b89db5 namespace: kserve-ci-e2e-test resourceVersion: "24709" uid: ac866c94-1013-41ae-a14a-d28e9b7c30f4 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: 'Created pod: success-200-isvc-a4170-predictor-c76b89db5-hv7sv' metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:21Z" name: success-200-isvc-a4170-predictor-c76b89db5.18a969d747dcf261 namespace: kserve-ci-e2e-test resourceVersion: "24738" uid: c5873b1c-6e56-4bc9-a803-2b0f893ce361 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24706" uid: ecaeee8b-fa0d-4d49-8a60-2f4977f5f966 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: Scaled up replica set success-200-isvc-a4170-predictor-c76b89db5 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:21Z" name: success-200-isvc-a4170-predictor.18a969d7434d5a95 namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 25cc51ae-6e83-4897-9652-a34f4991ec57 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: 4ca9874d-2ce7-40d5-b97b-fb216af979b3 kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-a4170-predictor.18a969dac1d341e5 namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: 6467f4ab-6719-4ffb-bed4-d82466cf5f26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: 4ca9874d-2ce7-40d5-b97b-fb216af979b3 kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-a4170-predictor.18a969dac2310fcb namespace: kserve-ci-e2e-test resourceVersion: "25126" uid: c37c5a8d-d5b0-4364-99f8-0ee5c84d91f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 4ca9874d-2ce7-40d5-b97b-fb216af979b3 kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:21Z" name: success-200-isvc-a4170-predictor.18a969e1c1505d19 namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: 8c229e19-b5a9-41c6-a937-2ec3bc31a74b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a4170-predictor namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 4ca9874d-2ce7-40d5-b97b-fb216af979b3 kind: Event lastTimestamp: "2026-04-24T21:59:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:21Z" name: success-200-isvc-a4170-predictor.18a969e1c23414c2 namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 5b5e5a0e-c30c-4f5e-9513-e2dacaf3e789 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a4170 namespace: kserve-ci-e2e-test resourceVersion: "24686" uid: 4e9a335a-ff21-46aa-97cd-e20df817cf12 kind: Event lastTimestamp: "2026-04-24T21:59:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:10Z" name: success-200-isvc-a4170.18a969d74317818d namespace: kserve-ci-e2e-test resourceVersion: "25614" uid: 2e3a98cf-ac46-4192-9fbf-79b4b9f9a1f9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a4170 namespace: kserve-ci-e2e-test resourceVersion: "24686" uid: 4e9a335a-ff21-46aa-97cd-e20df817cf12 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-a4170": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a4170": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:58:21Z" name: success-200-isvc-a4170.18a969d743735ab5 namespace: kserve-ci-e2e-test resourceVersion: "24714" uid: fcee32a2-c6a5-4cc4-8942-21810ca180b6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a4170 namespace: kserve-ci-e2e-test resourceVersion: "24686" uid: 4e9a335a-ff21-46aa-97cd-e20df817cf12 kind: Event lastTimestamp: "2026-04-24T21:58:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a4170": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:58:21Z" name: success-200-isvc-a4170.18a969d74373aeae namespace: kserve-ci-e2e-test resourceVersion: "24715" uid: a37571c3-bb41-4075-8215-08b9b28f9209 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a4170 namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: 4e9a335a-ff21-46aa-97cd-e20df817cf12 kind: Event lastTimestamp: "2026-04-24T21:59:10Z" message: InferenceService [success-200-isvc-a4170] is Ready metadata: creationTimestamp: "2026-04-24T21:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:10Z" name: success-200-isvc-a4170.18a969e2a2b0be1c namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: 43ff8fee-e86d-4c11-a3af-b4ab42299bc6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:48:26.014828Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b9878-predictor-fb4f998b7-crfr7 to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:48:26Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694cacf97c74 namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 64dd2758-6ece-4115-884f-1781336a816a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b9878-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:26Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694cb5f4df45 namespace: kserve-ci-e2e-test resourceVersion: "19874" uid: 92b29255-ec45-4718-8ef2-b03c07b822ae reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19841" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:48:27Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:48:27Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694ceaaf82fe namespace: kserve-ci-e2e-test resourceVersion: "19898" uid: 887bc42d-477e-4c84-af84-ee3acb8454da reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:48:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:27Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694cec18bda3 namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: d1474023-6cfd-4c54-a640-b13c7e4b8e91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:48:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:27Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694cf00c8c57 namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: efbbd33c-670f-4294-9af6-1d6c4478809e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:48:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:27Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694cf07ffbe8 namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: 069f1caa-857a-4963-b290-89c76f5abdad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:48:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:27Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694cf08d0695 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 91d540dc-335a-4f82-800e-7473e1b307b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:48:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:27Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694cf3e50262 namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: da6f9eab-58f5-4841-ae60-a3ddc16ed9b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:48:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:27Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694cf45940c8 namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: 7e15d299-3589-4067-9866-813411446f8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:48:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:49:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:04Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9694d53fccfd1 namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 15685269-e46d-4fb7-84f0-5a4f2a2a8f03 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9695d05768e8e namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 4f9dde53-3555-4601-af6c-1df57df87926 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:49:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:36Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9695d057788ce namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: f718ec63-037d-4c34-8cac-d15013cdc0d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 95c6330c-680a-4894-ab45-4566cb30ef16 kind: Event lastTimestamp: "2026-04-24T21:49:39Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:39Z" name: success-200-isvc-b9878-predictor-fb4f998b7-crfr7.18a9695ddc11a3b2 namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: 534add2f-ce76-44ab-b103-00a37aa12cd4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b9878-predictor-fb4f998b7 namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: 8344f2fb-d63a-4d6c-a9bb-002635fecaee kind: Event lastTimestamp: "2026-04-24T21:48:26Z" message: 'Created pod: success-200-isvc-b9878-predictor-fb4f998b7-crfr7' metadata: creationTimestamp: "2026-04-24T21:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:26Z" name: success-200-isvc-b9878-predictor-fb4f998b7.18a9694cac464981 namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: 667c02c9-ce69-453f-8b75-9b44338c4fcb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: bab1a688-d15d-4b11-a0bf-4a56d77444eb kind: Event lastTimestamp: "2026-04-24T21:48:25Z" message: Scaled up replica set success-200-isvc-b9878-predictor-fb4f998b7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:25Z" name: success-200-isvc-b9878-predictor.18a9694caa1933c1 namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 6a69b5e2-eb0b-4711-8530-b7e816a18404 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:48:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "19810" uid: fae81ad7-47a8-4f33-aeca-3c16d195b075 kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: success-200-isvc-b9878-predictor.18a96950297fc29e namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 286309ca-a180-4068-9b4c-aff24f0e782f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:48:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "19810" uid: fae81ad7-47a8-4f33-aeca-3c16d195b075 kind: Event lastTimestamp: "2026-04-24T21:49:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:48:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:26Z" name: success-200-isvc-b9878-predictor.18a9695029e95326 namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 2c9b6bfd-c09f-4671-8c05-5e34d60d0182 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: fae81ad7-47a8-4f33-aeca-3c16d195b075 kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-b9878-predictor.18a96953a891be1b namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: f3bf3aad-281f-4351-8bac-afd0cab023c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9878-predictor namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: fae81ad7-47a8-4f33-aeca-3c16d195b075 kind: Event lastTimestamp: "2026-04-24T21:48:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:48:56Z" name: success-200-isvc-b9878-predictor.18a96953a8dd0ac0 namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: c29a6257-f3f8-4f7e-bd30-56591992801b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:48:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9878 namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 847dbd56-355f-4479-ac9f-287950e13059 kind: Event lastTimestamp: "2026-04-24T21:49:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:14Z" name: success-200-isvc-b9878.18a9694caa2129e0 namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 625e9c13-4bd2-453c-b49f-9bb10eb17e52 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9878 namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 847dbd56-355f-4479-ac9f-287950e13059 kind: Event lastTimestamp: "2026-04-24T21:48:25Z" message: 'Failed to update status for InferenceService "success-200-isvc-b9878": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b9878": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:25Z" name: success-200-isvc-b9878.18a9694caaac78bb namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: e8ff4c23-3a18-4e0c-9048-b2299d99ceee reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:48:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9878 namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 847dbd56-355f-4479-ac9f-287950e13059 kind: Event lastTimestamp: "2026-04-24T21:48:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b9878": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:48:25Z" name: success-200-isvc-b9878.18a9694caaace4f5 namespace: kserve-ci-e2e-test resourceVersion: "19818" uid: 976e4a1d-7320-4884-94f5-04c4ec508cd9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9878 namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: 847dbd56-355f-4479-ac9f-287950e13059 kind: Event lastTimestamp: "2026-04-24T21:49:14Z" message: InferenceService [success-200-isvc-b9878] is Ready metadata: creationTimestamp: "2026-04-24T21:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:14Z" name: success-200-isvc-b9878.18a969580f2d6d94 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 88b59f15-8d7e-4dda-97d1-16faaaba7dc4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:38:31.232323Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c2312cc3d1 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 9b92e242-193e-44eb-8acf-3a7c93eba988 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c12fa-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c2392d9812 namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: c99a67e9-789a-46b6-a529-4c47e6c0cbb0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:38:32Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c26e93414f namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: c2269d92-d29e-40c6-8787-91b89ed4bbb1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c2704ab1a2 namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: 09569309-4d49-456f-96b0-2396b0cc7573 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c273907030 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: 536a0ec0-d935-415f-9016-e99dd3120455 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c2740db7c5 namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 496c6531-b42d-4c76-942f-0c550f652eaf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c2741b42ff namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: a658e3d8-797d-4620-9e89-098d9e109a71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c278b3d573 namespace: kserve-ci-e2e-test resourceVersion: "15059" uid: 71ef37cf-efc0-4b7b-a241-d68c0205309c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:38:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:32Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c279905c99 namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 43491f40-811a-4aa1-9b9f-3b2857ea3ffc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:47:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:48Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a968c28ab10170 namespace: kserve-ci-e2e-test resourceVersion: "19480" uid: 4f25c6c7-79f8-4073-8b55-0f82da04a405 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:46Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a9694361450bae namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: 5efe291a-0238-4eb7-a92e-55b16e181ebf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:47:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:47:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:46Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a969436145d17d namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 114e3446-393a-433c-8d0d-cd00d2b933b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c57b7cd6-ea5d-4873-860c-1506f1b5eab9 kind: Event lastTimestamp: "2026-04-24T21:47:48Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:48Z" name: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb.18a96943ff189330 namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: b5ebe1ae-1fd7-422f-ab2a-d69ed5bdfcf6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c12fa-predictor-5fb79c554f namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 908378e6-aa4b-493c-84c2-e8ca87aa1ffc kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: 'Created pod: success-200-isvc-c12fa-predictor-5fb79c554f-vpzgb' metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-c12fa-predictor-5fb79c554f.18a968c23096a738 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 3b579f24-d987-4abd-bfd1-93d9428c9f06 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: b922f7fc-6441-4da8-8033-e0685a5c6d7f kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: Scaled up replica set success-200-isvc-c12fa-predictor-5fb79c554f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-c12fa-predictor.18a968c22ef3d222 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: f31a5dd1-3dcd-42aa-89d9-d4a06d8bb4ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: aa6c1220-8e98-4481-b5d8-15c61a9d236a kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:01Z" name: success-200-isvc-c12fa-predictor.18a968c5ad73243c namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 2a15f85f-0670-4391-98e0-770716d63c71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: aa6c1220-8e98-4481-b5d8-15c61a9d236a kind: Event lastTimestamp: "2026-04-24T21:39:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:01Z" name: success-200-isvc-c12fa-predictor.18a968c5add826b9 namespace: kserve-ci-e2e-test resourceVersion: "15370" uid: 66fc8233-a2e7-47ce-95ef-9e5da6bf206b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:39:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: aa6c1220-8e98-4481-b5d8-15c61a9d236a kind: Event lastTimestamp: "2026-04-24T21:39:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:46Z" name: success-200-isvc-c12fa-predictor.18a968ccab46a125 namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: b37cc707-b523-4c67-ae25-0a9de467133a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:39:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c12fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: aa6c1220-8e98-4481-b5d8-15c61a9d236a kind: Event lastTimestamp: "2026-04-24T21:39:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:46Z" name: success-200-isvc-c12fa-predictor.18a968ccab918771 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: f6879b0c-6caa-4852-b72e-b05fac0f3bc6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c12fa namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 879927d8-3fd5-4efc-8a6b-7426e72bab58 kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:18Z" name: success-200-isvc-c12fa.18a968c22ed6331f namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 629dffb4-c041-4478-b7d7-4ea45a63b9a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c12fa namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 879927d8-3fd5-4efc-8a6b-7426e72bab58 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-c12fa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c12fa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-c12fa.18a968c22f5e0d85 namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: 7cf96d12-728f-407d-9a9a-d8931666a699 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c12fa namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 879927d8-3fd5-4efc-8a6b-7426e72bab58 kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c12fa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:31Z" name: success-200-isvc-c12fa.18a968c22f5e61a6 namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: c4b51040-9247-420e-9cca-d3d6950e70bd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c12fa namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: 879927d8-3fd5-4efc-8a6b-7426e72bab58 kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: InferenceService [success-200-isvc-c12fa] is Ready metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:18Z" name: success-200-isvc-c12fa.18a968cd4b20ec13 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 96bd2251-9b85-446f-bb9c-0f8a78fe8e57 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:58:51.189134Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25088" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 to ip-10-0-136-201.ec2.internal metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969de3c447118 namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: 6f49eec8-ae15-4499-b190-0cc84824ce8c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f4deb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969de4d385d21 namespace: kserve-ci-e2e-test resourceVersion: "25145" uid: 4b84d887-1af1-4e49-b56a-53716bdfe34f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: Add eth0 [10.134.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:58:52Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969de79eaf531 namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3f119759-398c-4fb2-bff8-afb7bf05b8b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1449" already present on machine metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969de7b91891f namespace: kserve-ci-e2e-test resourceVersion: "25171" uid: 1cadcbd8-0188-4182-ab94-ad81be207c2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969de7f801d48 namespace: kserve-ci-e2e-test resourceVersion: "25174" uid: 00680de7-fa5e-4212-a64c-b517542872e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969de7ff66b0d namespace: kserve-ci-e2e-test resourceVersion: "25175" uid: 848423dc-7645-49bf-9587-5dc3a5c75248 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969de8002d3ba namespace: kserve-ci-e2e-test resourceVersion: "25176" uid: 3e244319-fd2e-43fc-afdd-3f30a42c50a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969de83d96d5f namespace: kserve-ci-e2e-test resourceVersion: "25177" uid: 0cd5380a-d0c5-461c-9731-4a3387774cbd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T21:58:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:58:52Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969de8457851e namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: 3851ffe6-6304-4e8d-8045-6a2139bb36cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:58:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T21:59:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.52:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:30Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a969dee62de783 namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: c98acbea-d730-4976-8ab1-2d3adb0764d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T22:16:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:16:10Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a96ad04264df82 namespace: kserve-ci-e2e-test resourceVersion: "33126" uid: 30ff2622-211d-4eee-a284-958fbbdefe01 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: ab9fee25-cf0b-4243-9b8d-702558af3735 kind: Event lastTimestamp: "2026-04-24T22:16:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:16:10Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9.18a96ad04265486b namespace: kserve-ci-e2e-test resourceVersion: "33127" uid: a7737fec-425b-4b6c-9ad3-33a041f91553 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-201.ec2.internal source: component: kubelet host: ip-10-0-136-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f4deb-predictor-55df7bbd7c namespace: kserve-ci-e2e-test resourceVersion: "25077" uid: dd1b3429-ebe2-42df-a09f-d64742d12360 kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: 'Created pod: success-200-isvc-f4deb-predictor-55df7bbd7c-rd4c9' metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-f4deb-predictor-55df7bbd7c.18a969de3bb7a6f6 namespace: kserve-ci-e2e-test resourceVersion: "25093" uid: 0736ca7e-802b-4856-9169-a03b4ac7b634 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: 517b1683-f38d-46e7-b9af-d72276d92d2a kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: Scaled up replica set success-200-isvc-f4deb-predictor-55df7bbd7c from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-f4deb-predictor.18a969de3964eba3 namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: 201f8459-b637-4690-ac0b-9ebd2d62d010 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25078" uid: ca9183bd-8295-4548-850e-b5d5605d2ba0 kind: Event lastTimestamp: "2026-04-24T21:59:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:06Z" name: success-200-isvc-f4deb-predictor.18a969e1b862616c namespace: kserve-ci-e2e-test resourceVersion: "25554" uid: 75068178-2176-4d29-a65f-dd2630caa9e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25078" uid: ca9183bd-8295-4548-850e-b5d5605d2ba0 kind: Event lastTimestamp: "2026-04-24T21:59:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:06Z" name: success-200-isvc-f4deb-predictor.18a969e1b8c69863 namespace: kserve-ci-e2e-test resourceVersion: "25556" uid: 4f0fa4a9-75cd-4fa1-8fa8-e18c0a9d8a17 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25555" uid: ca9183bd-8295-4548-850e-b5d5605d2ba0 kind: Event lastTimestamp: "2026-04-24T22:00:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:06Z" name: success-200-isvc-f4deb-predictor.18a969e5381d7416 namespace: kserve-ci-e2e-test resourceVersion: "26340" uid: 329a3280-ec25-4c64-be45-ef187ac8e3db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f4deb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25555" uid: ca9183bd-8295-4548-850e-b5d5605d2ba0 kind: Event lastTimestamp: "2026-04-24T22:00:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:06Z" name: success-200-isvc-f4deb-predictor.18a969e538800d3d namespace: kserve-ci-e2e-test resourceVersion: "26341" uid: e8beeccb-dee7-4964-b6cf-328da3604db3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f4deb namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: c7f01306-8b27-4636-976e-0f3d8971470a kind: Event lastTimestamp: "2026-04-24T21:59:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:40Z" name: success-200-isvc-f4deb.18a969de398881b4 namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: 16d99758-d154-4c5d-af9f-411369a758d5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f4deb namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: c7f01306-8b27-4636-976e-0f3d8971470a kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-f4deb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f4deb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-f4deb.18a969de3a572784 namespace: kserve-ci-e2e-test resourceVersion: "25083" uid: de3d61ee-58b7-4cc4-9f32-1fc0249891a3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f4deb namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: c7f01306-8b27-4636-976e-0f3d8971470a kind: Event lastTimestamp: "2026-04-24T21:58:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f4deb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:58:51Z" name: success-200-isvc-f4deb.18a969de3a5774dc namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: 7b8a4ff8-76d1-4ddf-a846-9040a4ffd882 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f4deb namespace: kserve-ci-e2e-test resourceVersion: "26060" uid: c7f01306-8b27-4636-976e-0f3d8971470a kind: Event lastTimestamp: "2026-04-24T21:59:40Z" message: InferenceService [success-200-isvc-f4deb] is Ready metadata: creationTimestamp: "2026-04-24T21:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:40Z" name: success-200-isvc-f4deb.18a969e9a22f640f namespace: kserve-ci-e2e-test resourceVersion: "26061" uid: 19cba72a-c585-47dc-a17a-2710b441f9ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:38:20.895317Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-914c2-55b54cc8f8-kv6sm namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: b2864ba5-2503-4c9d-b4a6-da3a6e5c896c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-914c2-55b54cc8f8-kv6sm to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:38:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:38:20Z" name: switch-graph-914c2-55b54cc8f8-kv6sm.18a968bfc90a943d namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: 76f1e32c-8031-4752-9a34-95bc10fb79f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-914c2-55b54cc8f8-kv6sm namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: b2864ba5-2503-4c9d-b4a6-da3a6e5c896c kind: Event lastTimestamp: "2026-04-24T21:38:21Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:38:21Z" name: switch-graph-914c2-55b54cc8f8-kv6sm.18a968bfe2487e61 namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 66ef0fe4-c938-4eb0-b103-0846435324c9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-914c2} kind: Pod name: switch-graph-914c2-55b54cc8f8-kv6sm namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: b2864ba5-2503-4c9d-b4a6-da3a6e5c896c kind: Event lastTimestamp: "2026-04-24T21:38:21Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" metadata: creationTimestamp: "2026-04-24T21:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:21Z" name: switch-graph-914c2-55b54cc8f8-kv6sm.18a968bfe392cd86 namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 210dde07-5584-49f9-a426-1373e829b7c0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-914c2} kind: Pod name: switch-graph-914c2-55b54cc8f8-kv6sm namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: b2864ba5-2503-4c9d-b4a6-da3a6e5c896c kind: Event lastTimestamp: "2026-04-24T21:38:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" in 1.812s (1.812s including waiting). Image size: 216346205 bytes.' metadata: creationTimestamp: "2026-04-24T21:38:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:23Z" name: switch-graph-914c2-55b54cc8f8-kv6sm.18a968c04f9d45cd namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: 5e4ae21c-2c3a-4123-9412-f910584d3f82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-914c2} kind: Pod name: switch-graph-914c2-55b54cc8f8-kv6sm namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: b2864ba5-2503-4c9d-b4a6-da3a6e5c896c kind: Event lastTimestamp: "2026-04-24T21:38:23Z" message: 'Created container: switch-graph-914c2' metadata: creationTimestamp: "2026-04-24T21:38:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:23Z" name: switch-graph-914c2-55b54cc8f8-kv6sm.18a968c053f6fc6b namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: f11907ec-912c-4305-9197-d8afd95bad7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-914c2} kind: Pod name: switch-graph-914c2-55b54cc8f8-kv6sm namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: b2864ba5-2503-4c9d-b4a6-da3a6e5c896c kind: Event lastTimestamp: "2026-04-24T21:38:23Z" message: Started container switch-graph-914c2 metadata: creationTimestamp: "2026-04-24T21:38:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:23Z" name: switch-graph-914c2-55b54cc8f8-kv6sm.18a968c0546d2c9c namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 31d2711f-2e21-4980-923c-a6009d128b73 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-914c2} kind: Pod name: switch-graph-914c2-55b54cc8f8-kv6sm namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: b2864ba5-2503-4c9d-b4a6-da3a6e5c896c kind: Event lastTimestamp: "2026-04-24T21:38:31Z" message: Stopping container switch-graph-914c2 metadata: creationTimestamp: "2026-04-24T21:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:31Z" name: switch-graph-914c2-55b54cc8f8-kv6sm.18a968c22706baeb namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 4231170a-3e1e-4ff3-804d-a82b36920c6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-914c2} kind: Pod name: switch-graph-914c2-55b54cc8f8-kv6sm namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: b2864ba5-2503-4c9d-b4a6-da3a6e5c896c kind: Event lastTimestamp: "2026-04-24T21:38:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:38:59Z" name: switch-graph-914c2-55b54cc8f8-kv6sm.18a968c2e8d71cdf namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: ce4b8f17-b64c-4bfb-9d12-6f2265fa8550 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-914c2-55b54cc8f8 namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 3d43f208-6b47-4304-9875-68890e564576 kind: Event lastTimestamp: "2026-04-24T21:38:20Z" message: 'Created pod: switch-graph-914c2-55b54cc8f8-kv6sm' metadata: creationTimestamp: "2026-04-24T21:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:20Z" name: switch-graph-914c2-55b54cc8f8.18a968bfc87957c2 namespace: kserve-ci-e2e-test resourceVersion: "14819" uid: 8d07c363-a152-4757-a902-ce4f6c7f4760 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-914c2 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 36456b50-9425-43b9-a729-6d2a8e3674d6 kind: Event lastTimestamp: "2026-04-24T21:38:20Z" message: Scaled up replica set switch-graph-914c2-55b54cc8f8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:38:20Z" name: switch-graph-914c2.18a968bfc7aaf735 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: ac7496d2-a458-4dde-81e2-be9aea39ca29 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-914c2 namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: ae21e781-124f-4a1f-9347-5014425e4620 kind: Event lastTimestamp: "2026-04-24T21:38:20Z" message: InferenceGraph [switch-graph-914c2] is Ready metadata: creationTimestamp: "2026-04-24T21:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:20Z" name: switch-graph-914c2.18a968bfcda358f4 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: e3507f41-759d-43e4-abca-3952b158f694 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-914c2 namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: ae21e781-124f-4a1f-9347-5014425e4620 kind: Event lastTimestamp: "2026-04-24T21:38:21Z" message: 'Failed to update status for InferenceGraph "switch-graph-914c2": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-914c2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:21Z" name: switch-graph-914c2.18a968bfd0ab842b namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: 194a0226-8983-4cbf-a8b5-74acf4992a65 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:38:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-914c2 namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: ae21e781-124f-4a1f-9347-5014425e4620 kind: Event lastTimestamp: "2026-04-24T21:38:21Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-914c2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:38:21Z" name: switch-graph-914c2.18a968bfd0abe599 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 93adf5e3-c5e8-4ebd-b856-6be06cbe384a reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:31.300340Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c12fa-594b845bff-sdhrf namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: b4b83589-a966-4c69-b88a-88989d6eed00 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-c12fa-594b845bff-sdhrf to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:31Z" name: switch-graph-c12fa-594b845bff-sdhrf.18a968d02d820a3c namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 6f294774-c957-4c1d-b67f-8a468064f6d5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c12fa-594b845bff-sdhrf namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: b4b83589-a966-4c69-b88a-88989d6eed00 kind: Event lastTimestamp: "2026-04-24T21:39:31Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:31Z" name: switch-graph-c12fa-594b845bff-sdhrf.18a968d047664f75 namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: 168aaded-7c9b-4785-82ca-407b8fae27fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c12fa} kind: Pod name: switch-graph-c12fa-594b845bff-sdhrf namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: b4b83589-a966-4c69-b88a-88989d6eed00 kind: Event lastTimestamp: "2026-04-24T21:39:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:31Z" name: switch-graph-c12fa-594b845bff-sdhrf.18a968d04958f5cf namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 77a41a9f-fbd1-45cd-b1b1-cd2dd8e28b73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c12fa} kind: Pod name: switch-graph-c12fa-594b845bff-sdhrf namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: b4b83589-a966-4c69-b88a-88989d6eed00 kind: Event lastTimestamp: "2026-04-24T21:39:31Z" message: 'Created container: switch-graph-c12fa' metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:31Z" name: switch-graph-c12fa-594b845bff-sdhrf.18a968d04cb76804 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 62aa3a0f-6cbb-4667-93e3-9ae4f157bab1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c12fa} kind: Pod name: switch-graph-c12fa-594b845bff-sdhrf namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: b4b83589-a966-4c69-b88a-88989d6eed00 kind: Event lastTimestamp: "2026-04-24T21:39:31Z" message: Started container switch-graph-c12fa metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:31Z" name: switch-graph-c12fa-594b845bff-sdhrf.18a968d04d30f5e0 namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: 3201335a-42d7-4df0-97d1-02fc3e44374c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:47:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c12fa} kind: Pod name: switch-graph-c12fa-594b845bff-sdhrf namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: b4b83589-a966-4c69-b88a-88989d6eed00 kind: Event lastTimestamp: "2026-04-24T21:47:45Z" message: Stopping container switch-graph-c12fa metadata: creationTimestamp: "2026-04-24T21:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:47:45Z" name: switch-graph-c12fa-594b845bff-sdhrf.18a969435a9b6395 namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: a337304b-fcd9-4d3e-968c-65d66ff53064 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:47:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c12fa} kind: Pod name: switch-graph-c12fa-594b845bff-sdhrf namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: b4b83589-a966-4c69-b88a-88989d6eed00 kind: Event lastTimestamp: "2026-04-24T21:48:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:48:13Z" name: switch-graph-c12fa-594b845bff-sdhrf.18a96943f275c864 namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 1fd027f6-dde8-4627-b7b3-95d4c038ad28 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-c12fa-594b845bff namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 53701cc6-3c4c-421b-9a3e-d58cc2f79c42 kind: Event lastTimestamp: "2026-04-24T21:39:31Z" message: 'Created pod: switch-graph-c12fa-594b845bff-sdhrf' metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:31Z" name: switch-graph-c12fa-594b845bff.18a968d02d15e427 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 908a3b76-88b0-4b6e-98f7-ad75709e32e0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-c12fa namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: ebed77be-c0da-4930-b92b-0c87421d4b99 kind: Event lastTimestamp: "2026-04-24T21:39:31Z" message: Scaled up replica set switch-graph-c12fa-594b845bff from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:31Z" name: switch-graph-c12fa.18a968d02c5e0483 namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: e53faa45-4564-4288-b278-f4d2e9412b2c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-c12fa namespace: kserve-ci-e2e-test resourceVersion: "15841" uid: e7b096c7-8654-4c86-b869-1181ad988895 kind: Event lastTimestamp: "2026-04-24T21:39:31Z" message: InferenceGraph [switch-graph-c12fa] is Ready metadata: creationTimestamp: "2026-04-24T21:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:31Z" name: switch-graph-c12fa.18a968d0323e4130 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: 3b7bd4fd-ce15-4885-b47f-fba613051a81 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c12fa namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: e8dec4b8-6e9c-454e-a915-0f35ec42c326 kind: Event lastTimestamp: "2026-04-24T21:40:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:01Z" name: switch-graph-c12fa.18a968d3aafeeb6f namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 8063e272-2995-4216-ac21-f4abbf7ea835 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c12fa namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: e8dec4b8-6e9c-454e-a915-0f35ec42c326 kind: Event lastTimestamp: "2026-04-24T21:40:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:01Z" name: switch-graph-c12fa.18a968d3ab7e51eb namespace: kserve-ci-e2e-test resourceVersion: "16116" uid: 1ea5a111-0cb4-48bd-9514-cfb84fd61ad1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:59:51.246335Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f4deb-6fd5579b57-5tldk namespace: kserve-ci-e2e-test resourceVersion: "26186" uid: 7b9a1e05-3423-4c72-aa72-f8c7e0e53360 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f4deb-6fd5579b57-5tldk to ip-10-0-133-48.ec2.internal metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:59:51Z" name: switch-graph-f4deb-6fd5579b57-5tldk.18a969ec37f490e9 namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: 32a08752-fe91-4c2f-a748-dc64a8857d18 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-57ffb78d4-nv5gm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f4deb-6fd5579b57-5tldk namespace: kserve-ci-e2e-test resourceVersion: "26196" uid: 7b9a1e05-3423-4c72-aa72-f8c7e0e53360 kind: Event lastTimestamp: "2026-04-24T21:59:51Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:59:51Z" name: switch-graph-f4deb-6fd5579b57-5tldk.18a969ec51b10891 namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: d6d1709a-8043-4425-947e-1f10939800d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f4deb} kind: Pod name: switch-graph-f4deb-6fd5579b57-5tldk namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 7b9a1e05-3423-4c72-aa72-f8c7e0e53360 kind: Event lastTimestamp: "2026-04-24T21:59:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b42fc184cd9997665c45700486ec1c9301c424ede94aa53d528583b0fe44a7ad" already present on machine metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:51Z" name: switch-graph-f4deb-6fd5579b57-5tldk.18a969ec5306f70f namespace: kserve-ci-e2e-test resourceVersion: "26211" uid: 62f4dbdb-f582-468a-93b1-4aa6720e39e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f4deb} kind: Pod name: switch-graph-f4deb-6fd5579b57-5tldk namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 7b9a1e05-3423-4c72-aa72-f8c7e0e53360 kind: Event lastTimestamp: "2026-04-24T21:59:51Z" message: 'Created container: switch-graph-f4deb' metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:51Z" name: switch-graph-f4deb-6fd5579b57-5tldk.18a969ec5688c69a namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: b37a324a-dc4a-4418-8154-fe2df8d35494 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f4deb} kind: Pod name: switch-graph-f4deb-6fd5579b57-5tldk namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 7b9a1e05-3423-4c72-aa72-f8c7e0e53360 kind: Event lastTimestamp: "2026-04-24T21:59:51Z" message: Started container switch-graph-f4deb metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:59:51Z" name: switch-graph-f4deb-6fd5579b57-5tldk.18a969ec5707af2e namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: 8a2b3172-4642-42c0-962f-dfcd6773f1e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f4deb} kind: Pod name: switch-graph-f4deb-6fd5579b57-5tldk namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 7b9a1e05-3423-4c72-aa72-f8c7e0e53360 kind: Event lastTimestamp: "2026-04-24T22:16:10Z" message: Stopping container switch-graph-f4deb metadata: creationTimestamp: "2026-04-24T22:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:16:10Z" name: switch-graph-f4deb-6fd5579b57-5tldk.18a96ad03ae862bf namespace: kserve-ci-e2e-test resourceVersion: "33112" uid: 3a095b91-5e93-4137-87e4-be4201e3a540 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:16:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f4deb} kind: Pod name: switch-graph-f4deb-6fd5579b57-5tldk namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 7b9a1e05-3423-4c72-aa72-f8c7e0e53360 kind: Event lastTimestamp: "2026-04-24T22:16:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:16:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:16:33Z" name: switch-graph-f4deb-6fd5579b57-5tldk.18a96ad0da9b640d namespace: kserve-ci-e2e-test resourceVersion: "33347" uid: 704ea8f5-03bc-4866-a1b3-2b127e148e8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-48.ec2.internal source: component: kubelet host: ip-10-0-133-48.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f4deb-6fd5579b57 namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: 2ca981a6-6649-4442-a8f1-f6fc2e79b602 kind: Event lastTimestamp: "2026-04-24T21:59:51Z" message: 'Created pod: switch-graph-f4deb-6fd5579b57-5tldk' metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:51Z" name: switch-graph-f4deb-6fd5579b57.18a969ec37adc827 namespace: kserve-ci-e2e-test resourceVersion: "26189" uid: 11d92f48-5924-4170-bf28-3060f9c7b777 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f4deb namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: 68aa49ed-8661-496d-b242-01cdf1bd2440 kind: Event lastTimestamp: "2026-04-24T21:59:51Z" message: Scaled up replica set switch-graph-f4deb-6fd5579b57 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:59:51Z" name: switch-graph-f4deb.18a969ec36bc0c21 namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: 47f99d74-cfaa-421b-84c5-dc94cf71c4ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:59:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f4deb namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: b9382c43-ca61-468a-9567-0904a21f4d6b kind: Event lastTimestamp: "2026-04-24T21:59:51Z" message: InferenceGraph [switch-graph-f4deb] is Ready metadata: creationTimestamp: "2026-04-24T21:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:59:51Z" name: switch-graph-f4deb.18a969ec3b89ef5b namespace: kserve-ci-e2e-test resourceVersion: "26199" uid: b03d416d-311c-47a8-bbbb-3e454cda33cf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f4deb namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: e69609b7-5854-48ee-8c92-f4f6764b2990 kind: Event lastTimestamp: "2026-04-24T22:00:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:06Z" name: switch-graph-f4deb.18a969efb54ef63d namespace: kserve-ci-e2e-test resourceVersion: "26338" uid: a6a82e2a-8afd-40c5-ad54-51bba8b7788c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f4deb namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: e69609b7-5854-48ee-8c92-f4f6764b2990 kind: Event lastTimestamp: "2026-04-24T22:00:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:06Z" name: switch-graph-f4deb.18a969efb5b78e51 namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: 0da14c51-2c93-41de-b1ba-84a1c392507a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f4deb namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e69609b7-5854-48ee-8c92-f4f6764b2990 kind: Event lastTimestamp: "2026-04-24T22:00:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:21Z" name: switch-graph-f4deb.18a969f33463b49a namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 883faaeb-dd4f-4b80-a32c-8908d6e3d201 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f4deb namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e69609b7-5854-48ee-8c92-f4f6764b2990 kind: Event lastTimestamp: "2026-04-24T22:00:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:00:21Z" name: switch-graph-f4deb.18a969f334cee48b namespace: kserve-ci-e2e-test resourceVersion: "26476" uid: ec8cff30-ed3b-4dec-985b-28d86399d67e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33351"