--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:45:38.723628Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-614a5-7dcdd75c-qs5jf namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: bf0370bc-f2ce-4d77-bab2-057cf8adefd5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-614a5-7dcdd75c-qs5jf to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:45:38Z" name: ensemble-graph-614a5-7dcdd75c-qs5jf.18c6852ee312f3bd namespace: kserve-ci-e2e-test resourceVersion: "21563" uid: 249dc816-a39b-4205-b309-da65ef855853 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-614a5-7dcdd75c-qs5jf namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: bf0370bc-f2ce-4d77-bab2-057cf8adefd5 kind: Event lastTimestamp: "2026-07-28T17:45:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-614a5-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:39Z" name: ensemble-graph-614a5-7dcdd75c-qs5jf.18c6852ef42fd624 namespace: kserve-ci-e2e-test resourceVersion: "21586" uid: e59c7b43-b214-4df2-9385-4300b21fe757 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-614a5-7dcdd75c-qs5jf namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: bf0370bc-f2ce-4d77-bab2-057cf8adefd5 kind: Event lastTimestamp: "2026-07-28T17:45:39Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:45:39Z" name: ensemble-graph-614a5-7dcdd75c-qs5jf.18c6852f1ff69518 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: ce699ac1-b9f7-4d5f-b262-4f28431b0339 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-614a5} kind: Pod name: ensemble-graph-614a5-7dcdd75c-qs5jf namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: bf0370bc-f2ce-4d77-bab2-057cf8adefd5 kind: Event lastTimestamp: "2026-07-28T17:45:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:39Z" name: ensemble-graph-614a5-7dcdd75c-qs5jf.18c6852f2159c542 namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: d156797e-781b-48c1-bfa8-e0ad30107519 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-614a5} kind: Pod name: ensemble-graph-614a5-7dcdd75c-qs5jf namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: bf0370bc-f2ce-4d77-bab2-057cf8adefd5 kind: Event lastTimestamp: "2026-07-28T17:45:39Z" message: 'Created container: ensemble-graph-614a5' metadata: creationTimestamp: "2026-07-28T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:39Z" name: ensemble-graph-614a5-7dcdd75c-qs5jf.18c6852f243dc566 namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 31d59131-571d-4532-9070-38d06f7949d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-614a5} kind: Pod name: ensemble-graph-614a5-7dcdd75c-qs5jf namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: bf0370bc-f2ce-4d77-bab2-057cf8adefd5 kind: Event lastTimestamp: "2026-07-28T17:45:39Z" message: Started container ensemble-graph-614a5 metadata: creationTimestamp: "2026-07-28T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:39Z" name: ensemble-graph-614a5-7dcdd75c-qs5jf.18c6852f24c20df3 namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: 53828f6a-1b60-47d5-a9cb-ead6fc386071 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-614a5} kind: Pod name: ensemble-graph-614a5-7dcdd75c-qs5jf namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: bf0370bc-f2ce-4d77-bab2-057cf8adefd5 kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: Stopping container ensemble-graph-614a5 metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:53Z" name: ensemble-graph-614a5-7dcdd75c-qs5jf.18c685a212033067 namespace: kserve-ci-e2e-test resourceVersion: "25239" uid: 6fd4d662-7132-4e7c-bae0-1a676157cd8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-614a5} kind: Pod name: ensemble-graph-614a5-7dcdd75c-qs5jf namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: bf0370bc-f2ce-4d77-bab2-057cf8adefd5 kind: Event lastTimestamp: "2026-07-28T17:54:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:54:20Z" name: ensemble-graph-614a5-7dcdd75c-qs5jf.18c685a2a583a1c2 namespace: kserve-ci-e2e-test resourceVersion: "25586" uid: dd703aa5-a6a9-4008-9858-c0a7cc7b69a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-614a5-7dcdd75c namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: ea9b467c-5460-4187-a596-3fad690bf0d5 kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: 'Created pod: ensemble-graph-614a5-7dcdd75c-qs5jf' metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:38Z" name: ensemble-graph-614a5-7dcdd75c.18c6852ee274fc17 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: 75673b3c-775f-485e-9b80-14d3d639d07c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 8ae57461-7fc6-4ab6-ad9c-a82cd32b2601 kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: Scaled up replica set ensemble-graph-614a5-7dcdd75c from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:38Z" name: ensemble-graph-614a5.18c6852ee1700022 namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: 7a76645d-60d3-41e6-96a8-55a856e938d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21570" uid: 12c9f031-1ddd-4c62-a22f-3b3ec26728ec kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: InferenceGraph [ensemble-graph-614a5] is Ready metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:38Z" name: ensemble-graph-614a5.18c6852ee5bb1889 namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 329dfb6d-09a4-4f35-bf46-888f9d76fef9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: 12c9f031-1ddd-4c62-a22f-3b3ec26728ec kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-614a5": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-614a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:38Z" name: ensemble-graph-614a5.18c6852eea350ff9 namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: 05288a26-ec98-47ee-b2c3-e6b1ff5a67e3 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: 12c9f031-1ddd-4c62-a22f-3b3ec26728ec kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-614a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:38Z" name: ensemble-graph-614a5.18c6852eea356e57 namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: 77718339-da06-4826-aa0a-5295869c2f51 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21554" uid: 543fd532-85b3-4df7-8277-d881b1d3d914 kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:53Z" name: ensemble-graph-614a5.18c6853260034314 namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: 350b5bf9-6334-4065-a0cc-010f5a397195 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21554" uid: 543fd532-85b3-4df7-8277-d881b1d3d914 kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:53Z" name: ensemble-graph-614a5.18c6853260588f94 namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: 4f3a4730-2611-4e62-be85-2a2ca0f1a66c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21848" uid: 543fd532-85b3-4df7-8277-d881b1d3d914 kind: Event lastTimestamp: "2026-07-28T17:46:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:46:08Z" name: ensemble-graph-614a5.18c68535df102d65 namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: cece5ffa-83b6-4c47-ada1-b127202ec45b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21848" uid: 543fd532-85b3-4df7-8277-d881b1d3d914 kind: Event lastTimestamp: "2026-07-28T17:46:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:46:08Z" name: ensemble-graph-614a5.18c68535df836938 namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 01fc84bc-80ad-4612-bf76-3b52fedfaa4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:44:28.362953Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-8b8b5-846dd8bd57-vd26j namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: 5f51a692-a6d4-40f1-9588-2944c833a3e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-8b8b5-846dd8bd57-vd26j to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:44:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:44:28Z" name: ensemble-graph-8b8b5-846dd8bd57-vd26j.18c6851e8140208d namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: c6b98e1e-0df8-44ba-9e03-6a9e50fc3f4c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:28Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-8b8b5-846dd8bd57-vd26j namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5f51a692-a6d4-40f1-9588-2944c833a3e5 kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:44:28Z" name: ensemble-graph-8b8b5-846dd8bd57-vd26j.18c6851e9aabc980 namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: d1122bb4-0d21-4b27-ad89-ef089d65b52d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8b8b5} kind: Pod name: ensemble-graph-8b8b5-846dd8bd57-vd26j namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 5f51a692-a6d4-40f1-9588-2944c833a3e5 kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:28Z" name: ensemble-graph-8b8b5-846dd8bd57-vd26j.18c6851e9c7bbf2c namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 9830d141-466c-4b19-a4df-f2b29e00ed82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8b8b5} kind: Pod name: ensemble-graph-8b8b5-846dd8bd57-vd26j namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 5f51a692-a6d4-40f1-9588-2944c833a3e5 kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: 'Created container: ensemble-graph-8b8b5' metadata: creationTimestamp: "2026-07-28T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:28Z" name: ensemble-graph-8b8b5-846dd8bd57-vd26j.18c6851e9f4e0277 namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: 1729da6b-58ec-4900-8950-2ab7bb953893 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8b8b5} kind: Pod name: ensemble-graph-8b8b5-846dd8bd57-vd26j namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 5f51a692-a6d4-40f1-9588-2944c833a3e5 kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: Started container ensemble-graph-8b8b5 metadata: creationTimestamp: "2026-07-28T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:28Z" name: ensemble-graph-8b8b5-846dd8bd57-vd26j.18c6851e9fbe8db7 namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: 236707f7-d06b-4dce-aa93-8efa3193c049 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8b8b5} kind: Pod name: ensemble-graph-8b8b5-846dd8bd57-vd26j namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 5f51a692-a6d4-40f1-9588-2944c833a3e5 kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Stopping container ensemble-graph-8b8b5 metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:38Z" name: ensemble-graph-8b8b5-846dd8bd57-vd26j.18c68520da229335 namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: b3c93972-f8b6-4d76-b7c0-5d79e2371600 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8b8b5} kind: Pod name: ensemble-graph-8b8b5-846dd8bd57-vd26j namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 5f51a692-a6d4-40f1-9588-2944c833a3e5 kind: Event lastTimestamp: "2026-07-28T17:45:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:05Z" name: ensemble-graph-8b8b5-846dd8bd57-vd26j.18c685215c4eac22 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: aa5f4da3-b80b-4599-9c13-53b0ca3167bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-8b8b5-846dd8bd57 namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: 764972c4-8f04-48cc-a1de-ebe89cc35313 kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: 'Created pod: ensemble-graph-8b8b5-846dd8bd57-vd26j' metadata: creationTimestamp: "2026-07-28T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:28Z" name: ensemble-graph-8b8b5-846dd8bd57.18c6851e803f17a0 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 58efc3e8-3b52-4b5c-95b7-d2b43e7c6b5c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1e94e826-9143-4b03-b696-0bd4d6f7a93d kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: Scaled up replica set ensemble-graph-8b8b5-846dd8bd57 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:28Z" name: ensemble-graph-8b8b5.18c6851e7e191c5e namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 446e7298-8eef-4d2f-834c-5191ff6fd2f0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: 3aebab9d-0808-4d02-9988-c2725c1d785a kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: InferenceGraph [ensemble-graph-8b8b5] is Ready metadata: creationTimestamp: "2026-07-28T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:28Z" name: ensemble-graph-8b8b5.18c6851e872b4f32 namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: da871d65-1e37-478d-8bb4-26212d5aa137 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:34:13.589336Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:34:13Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848f5de339ef namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 9879d3c3-d322-4b4f-8042-94a27d8f5e7f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-22e0e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848f6cc77ae4 namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: 535d0369-31e0-4baa-92b4-139bb9a62324 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:34:14Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:34:14Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848f9b560979 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: c69b9b5c-ff0e-4235-8f5f-b09ff04ee4b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:34:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:14Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848f9ca4720d namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 267bd5d6-61bc-486f-a73c-1af1af4c0256 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:34:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:14Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848f9fe56a2e namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: b6c3d0e4-8830-49d9-ad74-89ad96349022 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:34:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:14Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848fa04ba563 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: efcaf11f-63e0-4102-99e7-87fb21e93956 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:34:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:14Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848fa05748c6 namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: dc89ff54-e61c-4cf3-a9b7-0d24cc91e038 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:34:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:14Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848fa3a7e7e7 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: dd7b593a-4c6d-401b-b585-72e2e183b80c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:34:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:14Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848fa40e1684 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: 9868a831-6010-4e3c-b549-d58bd50d0077 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:43:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:30Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6848fac17e50e namespace: kserve-ci-e2e-test resourceVersion: "20131" uid: ff55afc7-5b7e-420d-a4cd-a1bbf2fb24cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:28Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c6851091909596 namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: 8c0b8038-c663-4923-9606-377cc4ae5dac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:28Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c68510919113c4 namespace: kserve-ci-e2e-test resourceVersion: "20073" uid: 632b69d6-268a-418a-90c8-d0b49b0f5e42 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 040fce8d-7c1d-4b53-b4af-e6ddc6ebd1e9 kind: Event lastTimestamp: "2026-07-28T17:43:30Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:30Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k.18c685112080a156 namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: a9a4f048-ae5d-4994-a09f-3b16f5086bba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-22e0e-predictor-6bc85ccf58 namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: 42f63c30-3c2e-4887-ade9-5c61e4539930 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: 'Created pod: error-404-isvc-22e0e-predictor-6bc85ccf58-kfk2k' metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:13Z" name: error-404-isvc-22e0e-predictor-6bc85ccf58.18c6848f5d41f62b namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: baf48e37-04ef-45a4-8cab-0e4655077285 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 1648c9e8-fb30-4cff-aa7d-4900c007bd1d kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Scaled up replica set error-404-isvc-22e0e-predictor-6bc85ccf58 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:13Z" name: error-404-isvc-22e0e-predictor.18c6848f5b83cc78 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 83df53a4-674d-4058-a271-fa9cb418c2be reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 00abf8ea-2d28-4a7a-aac7-226c14520a26 kind: Event lastTimestamp: "2026-07-28T17:34:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:58Z" name: error-404-isvc-22e0e-predictor.18c68492da15a7c8 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 3bac295f-7e99-41c8-95b7-b6d8a7cc121a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 00abf8ea-2d28-4a7a-aac7-226c14520a26 kind: Event lastTimestamp: "2026-07-28T17:34:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:58Z" name: error-404-isvc-22e0e-predictor.18c68492da61d5a8 namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: 1cd07749-5502-4a88-8228-44f8ce34497f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:34:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 00abf8ea-2d28-4a7a-aac7-226c14520a26 kind: Event lastTimestamp: "2026-07-28T17:35:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:43Z" name: error-404-isvc-22e0e-predictor.18c684965948dbff namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: dfe42d94-609c-4e3a-8527-c1510ea5da6b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:34:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 00abf8ea-2d28-4a7a-aac7-226c14520a26 kind: Event lastTimestamp: "2026-07-28T17:35:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:43Z" name: error-404-isvc-22e0e-predictor.18c6849659c21e25 namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: 5bccd630-89d2-41e8-b15e-c3ce7880a5bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-22e0e namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 15eb5cc0-1332-4092-b73a-fd209d312303 kind: Event lastTimestamp: "2026-07-28T17:35:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:01Z" name: error-404-isvc-22e0e.18c6848f5b8a7bef namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: d9d7d548-46ca-411e-9a61-779fe04a03dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-22e0e namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 15eb5cc0-1332-4092-b73a-fd209d312303 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: 'Failed to update status for InferenceService "error-404-isvc-22e0e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-22e0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:13Z" name: error-404-isvc-22e0e.18c6848f5c282aa9 namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: 0babb4d2-0730-414b-92f2-807836c49259 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-22e0e namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 15eb5cc0-1332-4092-b73a-fd209d312303 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-22e0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:13Z" name: error-404-isvc-22e0e.18c6848f5c2877d9 namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: d3e84484-50ac-48c9-a209-7f2c72d0ab28 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-22e0e namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 15eb5cc0-1332-4092-b73a-fd209d312303 kind: Event lastTimestamp: "2026-07-28T17:35:00Z" message: InferenceService [error-404-isvc-22e0e] is Ready metadata: creationTimestamp: "2026-07-28T17:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:00Z" name: error-404-isvc-22e0e.18c6849a67588e16 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: 5de7a392-fbb6-4ae0-b41e-8006414961a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:44:39.281335Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-614a5-predictor-55c87cb68d-g9s48 to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c685210c0976d4 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: c779b719-06f3-4b4d-bb9e-c49133805db4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c6852125a5e118 namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: bf8a580f-cda6-442c-9150-ff9145136b58 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c68521284bfa04 namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: be9888ba-4788-4f07-9d17-f2a0eab9ed99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c685212b68ae50 namespace: kserve-ci-e2e-test resourceVersion: "20966" uid: 67c166c8-660d-49a5-b346-511afc8f568d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c685212c11f677 namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 05c07c36-1038-482c-9c32-c9e3a9944f09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c685212c1b21de namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: f5d23009-ad79-4f7f-9106-fed784aec9d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c685212fdde547 namespace: kserve-ci-e2e-test resourceVersion: "20969" uid: 225f4ca4-84de-4ab5-97b4-755e0507f08c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c68521306d0bb3 namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: 40ca3cf2-6fb5-4f28-86e7-ede2baf22895 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:53:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:55Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c6852131c67480 namespace: kserve-ci-e2e-test resourceVersion: "25386" uid: 5b855742-558f-4073-93a7-efd68e57e37f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:53Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c685a2288de746 namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: 112ed79e-55f4-4891-a4ef-297281ae7411 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:53Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c685a2288e606c namespace: kserve-ci-e2e-test resourceVersion: "25332" uid: ccdf2757-452b-482f-a8d2-44c9eda12fc7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 8dd18072-0f56-4a5b-8bfb-d06423530a14 kind: Event lastTimestamp: "2026-07-28T17:53:55Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": dial tcp 10.134.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:55Z" name: error-404-isvc-614a5-predictor-55c87cb68d-g9s48.18c685a2a6546308 namespace: kserve-ci-e2e-test resourceVersion: "25385" uid: d3808cfe-ecad-4b96-a44e-470ba5733a2b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-614a5-predictor-55c87cb68d namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: 63e24a14-bf34-464f-9c85-4707f65e3146 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: 'Created pod: error-404-isvc-614a5-predictor-55c87cb68d-g9s48' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor-55c87cb68d.18c6852109fb2424 namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 72acd5a4-40a7-4d82-ac30-41eb6c6b048e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: 6ffec1a9-5a33-4a2f-83e5-a391062f9dd6 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Scaled up replica set error-404-isvc-614a5-predictor-55c87cb68d from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-614a5-predictor.18c685210647627f namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 103cddc6-1b3b-4e25-85c7-203effb88d42 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: c0f4c785-6830-4425-8068-24cb1ac62cb8 kind: Event lastTimestamp: "2026-07-28T17:45:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:09Z" name: error-404-isvc-614a5-predictor.18c6852484b18ee0 namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: 267e3a4a-e8c8-49c4-8cf6-d37ee14865b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: c0f4c785-6830-4425-8068-24cb1ac62cb8 kind: Event lastTimestamp: "2026-07-28T17:45:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:09Z" name: error-404-isvc-614a5-predictor.18c685248522a7bb namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: 5ba7a78e-7cb4-4349-b41b-ed7bd28d088a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:45:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c0f4c785-6830-4425-8068-24cb1ac62cb8 kind: Event lastTimestamp: "2026-07-28T17:45:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:54Z" name: error-404-isvc-614a5-predictor.18c6852b82defb21 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: e08861d2-2fc7-4d19-b23d-bb940fc72e24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:45:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c0f4c785-6830-4425-8068-24cb1ac62cb8 kind: Event lastTimestamp: "2026-07-28T17:45:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:54Z" name: error-404-isvc-614a5-predictor.18c6852b835eab8d namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 9d4b2722-1911-4ba1-96e5-226c4bb00302 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-614a5 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 5c33d64b-b060-4dc0-935b-9f133b69167f kind: Event lastTimestamp: "2026-07-28T17:45:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:26Z" name: error-404-isvc-614a5.18c6852106686922 namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: ecc03445-8717-4b41-b7ce-6223b8cd4e01 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 5c33d64b-b060-4dc0-935b-9f133b69167f kind: Event lastTimestamp: "2026-07-28T17:45:26Z" message: InferenceService [error-404-isvc-614a5] is Ready metadata: creationTimestamp: "2026-07-28T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:26Z" name: error-404-isvc-614a5.18c6852beda76971 namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: d72e8080-8c9f-4201-bf37-e5a67b981827 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:44:38.463007Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:44:38Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c68520db42bd02 namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: 51da03a6-2d6d-447f-af22-f674bb8d01ab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:44:38Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c68520f672df9d namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 66b6afb6-3169-41d6-bf51-fcc0b8d74e45 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:38Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c68520f963efb8 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 961fdc50-4511-4da0-971c-75967797166a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c68520fd4988c6 namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: 9d17987e-a55c-4fe5-b01b-c6464c3eec52 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c68520fdba8534 namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: b7a91736-0f5d-47a6-bd95-e7cce81384b4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c68520fdc484e5 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: a37b7075-a1ff-4095-8a71-65394c883faf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c685210135962b namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: 20669e26-a8ef-4f09-a557-732b26bf9d6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c6852101bc762d namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: 6ee4304e-cadd-46ef-bf62-0d7611a7485c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:45:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:15Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c6852131b0abee namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 68279576-609c-44a2-abde-05ca00674c29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:52Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c685321e2f426a namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 0b1414f4-18b5-47db-9365-b77290b5cdc1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2ab8bb64-52b0-41a9-909c-b0b41258bea5 kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:52Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w.18c685321e300109 namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 26e64abf-ddb0-4023-878d-9f9bd9541b66 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-61a23-predictor-fd5b4c6d4 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 786dc946-af02-4d0e-965b-e9d696f3b092 kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: 'Created pod: error-404-isvc-61a23-predictor-fd5b4c6d4-52d4w' metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:38Z" name: error-404-isvc-61a23-predictor-fd5b4c6d4.18c68520da5a3dc7 namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: 8a67f429-4344-4329-94d5-f46828220750 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20778" uid: d604aa96-3cbe-4df9-9956-d9e5c1ecc9e7 kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Scaled up replica set error-404-isvc-61a23-predictor-fd5b4c6d4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:38Z" name: error-404-isvc-61a23-predictor.18c68520d7d35b1f namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: 9e552f79-0b2c-44ac-a2fb-0d3702562e21 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 32b94255-1370-4c43-8230-f08074e0d68f kind: Event lastTimestamp: "2026-07-28T17:45:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:08Z" name: error-404-isvc-61a23-predictor.18c68524565cf9c1 namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 0c467288-5612-4c92-b067-d22626363b2f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 32b94255-1370-4c43-8230-f08074e0d68f kind: Event lastTimestamp: "2026-07-28T17:45:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:08Z" name: error-404-isvc-61a23-predictor.18c6852456cb7401 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: 1b45ad8b-8b25-4502-9a80-4c04cebc8e80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:45:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 32b94255-1370-4c43-8230-f08074e0d68f kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:38Z" name: error-404-isvc-61a23-predictor.18c6852b54550a23 namespace: kserve-ci-e2e-test resourceVersion: "21540" uid: c787e46e-a278-475a-ab64-6e1d858db044 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:45:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 32b94255-1370-4c43-8230-f08074e0d68f kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:38Z" name: error-404-isvc-61a23-predictor.18c6852b54d2e252 namespace: kserve-ci-e2e-test resourceVersion: "21543" uid: cb9cc976-612d-4bad-82a4-a77ba9e432a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-61a23 namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 15117f76-cadd-46ec-ada2-034734fa195d kind: Event lastTimestamp: "2026-07-28T17:45:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:26Z" name: error-404-isvc-61a23.18c68520d836d112 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 57b1f4a0-86af-4fb0-92c5-380c124b91e2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-61a23 namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 15117f76-cadd-46ec-ada2-034734fa195d kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: 'Failed to update status for InferenceService "error-404-isvc-61a23": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-61a23": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:38Z" name: error-404-isvc-61a23.18c68520d8e843bc namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: d02b32f9-dd3d-4e40-b135-77bbdfeeb5eb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-61a23 namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 15117f76-cadd-46ec-ada2-034734fa195d kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-61a23": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:38Z" name: error-404-isvc-61a23.18c68520d8e89b42 namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 84ce8a95-6771-46a8-82e6-6aec8f2407b3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-61a23 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: 15117f76-cadd-46ec-ada2-034734fa195d kind: Event lastTimestamp: "2026-07-28T17:45:26Z" message: InferenceService [error-404-isvc-61a23] is Ready metadata: creationTimestamp: "2026-07-28T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:26Z" name: error-404-isvc-61a23.18c6852bf0aa20fb namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 2ede3c21-09d8-4e3d-a234-4de3746ecd9f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:45:52.640813Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-787a1-predictor-fcb464fdb-zzl47 to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:45:52Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c68532209a3f6e namespace: kserve-ci-e2e-test resourceVersion: "21803" uid: afe50746-e337-43d5-8cd0-502df5d9f0db reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21809" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:45:53Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c685323a618164 namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: 6d7cda35-88bc-49f6-b51b-ca5bb1915f6b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c685323cbb5cf3 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: 8eed661d-fae5-4e81-a267-d170a18cee6b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c68532406b4220 namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: 00161a5c-696f-417e-a467-258d2298c4f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c6853240e40898 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 094aa26d-5611-4cb8-834e-cf8aff51c21e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c6853240ee595d namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 94dc22e1-eb20-43e6-91c3-0879da8f8121 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c6853244ac8e80 namespace: kserve-ci-e2e-test resourceVersion: "21830" uid: af342043-9d2c-4bae-a92f-b970315268d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c68532452346cd namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: 66a339b5-b8f2-4ae6-b5cf-e1a60602cb25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:55:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:09Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c6853246240ebe namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 702df910-3686-4275-945c-97bc98eec6aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:07Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c685b356515c2e namespace: kserve-ci-e2e-test resourceVersion: "26514" uid: dc3997c0-19d5-407d-9324-3f58a1590c0f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:07Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c685b35653121c namespace: kserve-ci-e2e-test resourceVersion: "26515" uid: bccdf720-3dda-476f-aad0-08f5412824ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: 784170c7-f416-4d44-8c2d-ee625229e312 kind: Event lastTimestamp: "2026-07-28T17:55:09Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:09Z" name: error-404-isvc-787a1-predictor-fcb464fdb-zzl47.18c685b3baadee79 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 06eaa410-1e35-4cd2-a65c-f8423093a49a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-787a1-predictor-fcb464fdb namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: 1675a38d-9164-425d-bafb-1564f27bdb7f kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: 'Created pod: error-404-isvc-787a1-predictor-fcb464fdb-zzl47' metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:52Z" name: error-404-isvc-787a1-predictor-fcb464fdb.18c685321fe03f6f namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 132b7ea3-c22c-46d8-8e79-95cf0434ae97 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 53ba83cf-87b3-4283-baef-ad9dc2c3927e kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: Scaled up replica set error-404-isvc-787a1-predictor-fcb464fdb from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:52Z" name: error-404-isvc-787a1-predictor.18c685321dda67f3 namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: 5a603e81-4d1c-4304-b454-f5805c9f425b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:46:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: ef419807-1cb3-4b35-a1fb-014544cd3d3c kind: Event lastTimestamp: "2026-07-28T17:46:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:46:22Z" name: error-404-isvc-787a1-predictor.18c685359c5a4711 namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: 815e8414-3b4d-4fad-9395-496f11775d7d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:46:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: ef419807-1cb3-4b35-a1fb-014544cd3d3c kind: Event lastTimestamp: "2026-07-28T17:46:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:46:22Z" name: error-404-isvc-787a1-predictor.18c685359ce31cae namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: 77392089-d7e7-4652-b7c3-6ab543a0ae25 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:46:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: ef419807-1cb3-4b35-a1fb-014544cd3d3c kind: Event lastTimestamp: "2026-07-28T17:47:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:07Z" name: error-404-isvc-787a1-predictor.18c6853c9adcdef6 namespace: kserve-ci-e2e-test resourceVersion: "22473" uid: a734b2e0-1456-46da-8292-1edac24b70e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:46:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: ef419807-1cb3-4b35-a1fb-014544cd3d3c kind: Event lastTimestamp: "2026-07-28T17:47:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:07Z" name: error-404-isvc-787a1-predictor.18c6853c9b9d9ade namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: 724f6a49-708d-4840-b8fb-d1420fd58435 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-787a1 namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: 66746f50-bbc7-46d1-abd6-2fe1a1b5b214 kind: Event lastTimestamp: "2026-07-28T17:46:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:46:39Z" name: error-404-isvc-787a1.18c685321ddbaa84 namespace: kserve-ci-e2e-test resourceVersion: "22224" uid: 962220d9-6015-4cb1-af56-3878d3817161 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-787a1 namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: 66746f50-bbc7-46d1-abd6-2fe1a1b5b214 kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-787a1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-787a1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:52Z" name: error-404-isvc-787a1.18c685321e761b4e namespace: kserve-ci-e2e-test resourceVersion: "21792" uid: 16d4cc35-c201-4321-ad2f-3aaeb8978adb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-787a1 namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: 66746f50-bbc7-46d1-abd6-2fe1a1b5b214 kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-787a1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:52Z" name: error-404-isvc-787a1.18c685321e7689f3 namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 190d6560-421d-40e7-974d-2a906671edd5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-787a1 namespace: kserve-ci-e2e-test resourceVersion: "22222" uid: 66746f50-bbc7-46d1-abd6-2fe1a1b5b214 kind: Event lastTimestamp: "2026-07-28T17:46:39Z" message: InferenceService [error-404-isvc-787a1] is Ready metadata: creationTimestamp: "2026-07-28T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:46:39Z" name: error-404-isvc-787a1.18c6853d023eb667 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: bf389885-ad44-4718-8b05-cdee1c6bfe19 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:55:04.202458Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26334" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7d999-predictor-566585ff9d-rtwzr to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:55:04Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b28c3d12ff namespace: kserve-ci-e2e-test resourceVersion: "26341" uid: 5e4e33e1-fcad-4271-9642-216198c2d90b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T17:55:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7d999-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:04Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b2978c0889 namespace: kserve-ci-e2e-test resourceVersion: "26357" uid: 3caf2c3a-1cda-41e2-9283-9727b6eb64ba reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26347" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:55:05Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b2c9fc1316 namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 8941ad42-508b-4420-9ccf-efbda3bfc98f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b2cbc2ff91 namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: d1da4ae3-8c20-42fd-b4b5-7bdd5a81cd68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b2cf29ee31 namespace: kserve-ci-e2e-test resourceVersion: "26378" uid: e68acc2e-1ca8-4c34-ad62-92208d6a90c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b2cfa0b615 namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: 24a4432f-4005-40cf-ac3c-d1b5cc6d2a59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b2cfaa2687 namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: b401ceb4-2384-4504-a542-3db460387220 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b2d31b44f5 namespace: kserve-ci-e2e-test resourceVersion: "26382" uid: eb94af26-2a5a-4a12-a983-058d7dc44737 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b2d3bc952b namespace: kserve-ci-e2e-test resourceVersion: "26383" uid: 13b3f982-c98a-4840-b789-00d973671ac3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:55:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T17:55:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:42Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c685b32cf1d7ac namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: bc5b0579-0467-4d41-a02c-71cb60860bca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:04:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T18:04:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:04:18Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c68633b4722f71 namespace: kserve-ci-e2e-test resourceVersion: "30621" uid: b83f3017-7a69-4111-8eb7-31900e33afc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:04:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: e53e30d3-5717-4f68-b3c4-80fe2ae9be48 kind: Event lastTimestamp: "2026-07-28T18:04:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:04:18Z" name: error-404-isvc-7d999-predictor-566585ff9d-rtwzr.18c68633b472db62 namespace: kserve-ci-e2e-test resourceVersion: "30622" uid: 09f9bbd3-41a9-46cc-93d4-d4ebbfe39cf0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7d999-predictor-566585ff9d namespace: kserve-ci-e2e-test resourceVersion: "26317" uid: ff8cfd08-d475-433e-9a24-599b14a98985 kind: Event lastTimestamp: "2026-07-28T17:55:04Z" message: 'Created pod: error-404-isvc-7d999-predictor-566585ff9d-rtwzr' metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:04Z" name: error-404-isvc-7d999-predictor-566585ff9d.18c685b28b5e0eec namespace: kserve-ci-e2e-test resourceVersion: "26336" uid: 4136be97-7292-4530-8f05-a72a5e33ffe6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: bd26ff49-fc8b-4e6f-9a66-9d60415ac2e6 kind: Event lastTimestamp: "2026-07-28T17:55:04Z" message: Scaled up replica set error-404-isvc-7d999-predictor-566585ff9d from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:04Z" name: error-404-isvc-7d999-predictor.18c685b2886638a7 namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: e4cd7ab8-0699-44a3-bc13-932066d0804b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: c96aca27-1721-47c5-bb4a-49645f6192b3 kind: Event lastTimestamp: "2026-07-28T17:55:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:19Z" name: error-404-isvc-7d999-predictor.18c685b60712f769 namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: d2f7b811-f216-4bc9-a713-b089a51152b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: c96aca27-1721-47c5-bb4a-49645f6192b3 kind: Event lastTimestamp: "2026-07-28T17:55:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:19Z" name: error-404-isvc-7d999-predictor.18c685b60796b596 namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: 6e390173-7d7f-494c-a561-78642d13d171 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:55:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26661" uid: c96aca27-1721-47c5-bb4a-49645f6192b3 kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:04Z" name: error-404-isvc-7d999-predictor.18c685b986905b4b namespace: kserve-ci-e2e-test resourceVersion: "27081" uid: 3e63814a-1b0f-434f-97ac-862d0ee743d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:55:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26661" uid: c96aca27-1721-47c5-bb4a-49645f6192b3 kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:04Z" name: error-404-isvc-7d999-predictor.18c685b986f5d2a5 namespace: kserve-ci-e2e-test resourceVersion: "27085" uid: f8e214f3-efa1-4969-b4c4-c0d381fd1653 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7d999 namespace: kserve-ci-e2e-test resourceVersion: "26299" uid: 243f114d-41c3-4436-b9ca-4126d7005222 kind: Event lastTimestamp: "2026-07-28T17:55:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:53Z" name: error-404-isvc-7d999.18c685b28892c8ad namespace: kserve-ci-e2e-test resourceVersion: "26971" uid: 6377d2ba-d0fe-4bdf-9811-d1c0e3da7f9b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7d999 namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: 243f114d-41c3-4436-b9ca-4126d7005222 kind: Event lastTimestamp: "2026-07-28T17:55:53Z" message: InferenceService [error-404-isvc-7d999] is Ready metadata: creationTimestamp: "2026-07-28T17:55:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:53Z" name: error-404-isvc-7d999.18c685bdeb8f3ed9 namespace: kserve-ci-e2e-test resourceVersion: "26970" uid: 4a7c90d5-0861-48f6-84d9-f382034d1af8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:28.404853Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:28Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c685108b78279c namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 7196b787-6d81-4e13-acd6-d9218f15100d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8b8b5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:28Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68510990bfdb4 namespace: kserve-ci-e2e-test resourceVersion: "20081" uid: 07b080c1-11d7-4fc0-bc49-18d4dc33495a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:29Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68510c91c98d5 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 3da939bf-3a73-4602-93e7-d469c2ef9805 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68510cab0a5b6 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 4b497eb4-5709-4cbe-8235-47ba53fd534c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68510cdc6eb27 namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: cb782619-8816-4143-8c67-3d9974869129 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68510ce55fc1f namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 5e05d595-9b0c-4df9-b3ea-69e26d29d7e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68510ce5f6b61 namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 07733c72-26a5-4ab3-bcc1-0280cb7fe97c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68510d2218da6 namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: c734cc37-e205-41e8-8b5a-0c40bc54eeeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68510d2a8161e namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: dc00bb1b-0cd6-4419-83fb-781a8c7cb3ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:44:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:07Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68511483f2921 namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: ea1b701b-0b79-4a18-9345-684b90302f61 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c685210ef13e07 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: bb90eb44-ca2f-4636-8d9c-40a698284f72 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c685210ef1c97c namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: 20af261d-5f01-4e21-b8f6-8c2269298811 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 0abeb40e-8792-44da-a994-2ae33123547e kind: Event lastTimestamp: "2026-07-28T17:44:42Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:42Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5.18c68521d08b7b77 namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: 6ce12742-c933-46bf-b1fa-420a13b09c13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8b8b5-predictor-68f9fbd6b8 namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: 0e066b08-d890-4a53-979d-34394cc3b34e kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: 'Created pod: error-404-isvc-8b8b5-predictor-68f9fbd6b8-5djw5' metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:28Z" name: error-404-isvc-8b8b5-predictor-68f9fbd6b8.18c685108ab4b626 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: daefd429-96b8-404a-9d62-49aefabe8088 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: c34a7a48-871c-46e5-839a-ac650f03bac7 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: Scaled up replica set error-404-isvc-8b8b5-predictor-68f9fbd6b8 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:28Z" name: error-404-isvc-8b8b5-predictor.18c6851088618481 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 0f1c3a73-958a-457e-8804-2e190cb8b6ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: d60ae2b2-2d2d-4d9a-85c5-8bf7919dd667 kind: Event lastTimestamp: "2026-07-28T17:44:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:13Z" name: error-404-isvc-8b8b5-predictor.18c6851406ef3fd3 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: cc39fc16-c80a-4d86-857f-87ac20adeeae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: d60ae2b2-2d2d-4d9a-85c5-8bf7919dd667 kind: Event lastTimestamp: "2026-07-28T17:44:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:13Z" name: error-404-isvc-8b8b5-predictor.18c6851407610134 namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 58925b73-b57f-4674-8611-260018e9ed5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: d60ae2b2-2d2d-4d9a-85c5-8bf7919dd667 kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:28Z" name: error-404-isvc-8b8b5-predictor.18c6851786d7d1aa namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 8622bfaa-79f9-43ba-952e-74cb8c026099 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: d60ae2b2-2d2d-4d9a-85c5-8bf7919dd667 kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:28Z" name: error-404-isvc-8b8b5-predictor.18c6851787bc0f61 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 72ecd45c-06d3-440d-bfeb-b338a9f1d0ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: ee5425fc-6eb3-45e9-b098-4025c26c936b kind: Event lastTimestamp: "2026-07-28T17:44:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:17Z" name: error-404-isvc-8b8b5.18c6851088610d06 namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: 7b4e65f6-3786-4533-93bc-f399f4302aea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: ee5425fc-6eb3-45e9-b098-4025c26c936b kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-8b8b5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8b8b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:28Z" name: error-404-isvc-8b8b5.18c6851088fbd956 namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 3e9f8a28-e172-4c57-8e13-41be93880297 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: ee5425fc-6eb3-45e9-b098-4025c26c936b kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8b8b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:28Z" name: error-404-isvc-8b8b5.18c6851088fc2f79 namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: aba5c3cd-9f1c-4adf-9837-34da4c639038 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: ee5425fc-6eb3-45e9-b098-4025c26c936b kind: Event lastTimestamp: "2026-07-28T17:44:17Z" message: InferenceService [error-404-isvc-8b8b5] is Ready metadata: creationTimestamp: "2026-07-28T17:44:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:17Z" name: error-404-isvc-8b8b5.18c6851c04c024b8 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 7b982715-b720-452d-8412-cf0246c9545c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:35:23.434732Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:35:23Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c6849fa0ff71e8 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: f9278a42-e657-417b-8283-166d645c0e65 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:35:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b1dfe-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:24Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c6849fac2929e7 namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 3b1c4a98-9eff-4233-a4c6-28b76b5592d3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:35:25Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:35:25Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c684a013f5dbc4 namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 6deb5525-b6a5-479a-b2d5-0fe8bcba50d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:35:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:25Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c684a0154c46a0 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: 83692ff0-9252-4cf2-a761-0ca87382113c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:35:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:25Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c684a01846bb3d namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: e35df346-3fe0-44cd-af9c-f9158a740ae8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:35:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:25Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c684a018ab72d2 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: f142a8c2-ed36-4010-96f7-376af4c3758f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:35:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:25Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c684a018b59d0f namespace: kserve-ci-e2e-test resourceVersion: "16538" uid: 085ae2e3-32d4-4b5f-8e10-d84bd096cb27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:35:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:25Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c684a01b55d2b7 namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 91e657a6-2c04-482c-9ba4-abb65dfafb9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:35:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:25Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c684a01bba3c9d namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: 451eac1e-49bd-412a-a59b-22a517494d76 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:36:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:36:03Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c684a084d3160c namespace: kserve-ci-e2e-test resourceVersion: "16856" uid: 9b7aca2d-d9bc-4f5e-894b-7e508901b0ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:38Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c68520e1b64cf4 namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: bd858760-1347-4a3a-9f7c-a4bc717b45fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 74662a03-b254-4f2c-ac7c-d5d7328879e3 kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:38Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw.18c68520e1b6bb28 namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 128dd45f-3d6c-476a-9db9-9713245ad33c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b1dfe-predictor-54cdc9876b namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: d0c14c27-a59e-4f3e-ad75-5c33c4162fbc kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: 'Created pod: error-404-isvc-b1dfe-predictor-54cdc9876b-kc6bw' metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:23Z" name: error-404-isvc-b1dfe-predictor-54cdc9876b.18c6849fa07d2d02 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 1ea8ab83-ab99-4f8c-83d1-cb5bf63d3bc5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: c3d413a2-3542-480b-89b2-df41922334e3 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Scaled up replica set error-404-isvc-b1dfe-predictor-54cdc9876b from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:23Z" name: error-404-isvc-b1dfe-predictor.18c6849f9ecff7e8 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: a2696abc-96ff-4830-a3df-8e125805df63 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:35:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: ab39779b-25c5-4ca4-bfcb-bc8e7177af4f kind: Event lastTimestamp: "2026-07-28T17:36:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:08Z" name: error-404-isvc-b1dfe-predictor.18c684a31d77b9a7 namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: c20e1e21-7e7c-464f-8849-b53d4e3a6604 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:35:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: ab39779b-25c5-4ca4-bfcb-bc8e7177af4f kind: Event lastTimestamp: "2026-07-28T17:36:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:08Z" name: error-404-isvc-b1dfe-predictor.18c684a31dc1d69a namespace: kserve-ci-e2e-test resourceVersion: "16896" uid: 635513f6-5286-4223-b214-0419a4f76df1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:35:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: ab39779b-25c5-4ca4-bfcb-bc8e7177af4f kind: Event lastTimestamp: "2026-07-28T17:36:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:38Z" name: error-404-isvc-b1dfe-predictor.18c684a69c7ffda9 namespace: kserve-ci-e2e-test resourceVersion: "17170" uid: ff0b690b-9537-4fe5-9fc4-4dd7800a4229 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:35:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: ab39779b-25c5-4ca4-bfcb-bc8e7177af4f kind: Event lastTimestamp: "2026-07-28T17:36:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:38Z" name: error-404-isvc-b1dfe-predictor.18c684a69cddef5d namespace: kserve-ci-e2e-test resourceVersion: "17171" uid: 3ca12550-6b3b-4d9a-a417-d64d4158b4bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: 4ac57d87-4d73-4fef-8686-2fcaa07f4355 kind: Event lastTimestamp: "2026-07-28T17:36:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:36:13Z" name: error-404-isvc-b1dfe.18c6849f9f06502e namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: b56ec897-aec4-4020-88af-ac745a7a9a64 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: 4ac57d87-4d73-4fef-8686-2fcaa07f4355 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: 'Failed to update status for InferenceService "error-404-isvc-b1dfe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b1dfe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:23Z" name: error-404-isvc-b1dfe.18c6849f9f7b075f namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 08ee405d-dfac-424d-8d37-9e4905f23b34 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: 4ac57d87-4d73-4fef-8686-2fcaa07f4355 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b1dfe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:23Z" name: error-404-isvc-b1dfe.18c6849f9f7b4798 namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: d8f4b7c1-b887-498c-a689-34075f5a3726 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 4ac57d87-4d73-4fef-8686-2fcaa07f4355 kind: Event lastTimestamp: "2026-07-28T17:36:13Z" message: InferenceService [error-404-isvc-b1dfe] is Ready metadata: creationTimestamp: "2026-07-28T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:36:13Z" name: error-404-isvc-b1dfe.18c684ab4075234a namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: b0fff87f-98cf-4ef2-a8fa-87514bd26a09 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:55:07.518314Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26487" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:55:07Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c685b351e1099d namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: e3c705c4-aec5-473f-ba43-50287a4a1e97 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:55:08Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c685b36ea0168c namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: abbab87c-273c-4b03-ad83-801bdba9f19d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c685b3705cdddd namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: e3341b6f-2767-4e5d-992e-1e1fa4eec36b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c685b37336c0d9 namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: 8ca3f438-ce86-4a90-8f44-2bf2c101b64b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c685b373b10eb2 namespace: kserve-ci-e2e-test resourceVersion: "26535" uid: b1df9dd0-880f-484a-9b3d-307d6d034b46 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c685b373bcdd83 namespace: kserve-ci-e2e-test resourceVersion: "26536" uid: 3a27fa28-605d-46ee-bd41-377ff6ebcafa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c685b3768bef1b namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: e3c5d0ad-18c0-44d4-ae9b-831c6319df44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c685b377054467 namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: 385def07-5e94-4549-8bd5-e7f95af75a04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T17:55:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:45Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c685b3e12f38de namespace: kserve-ci-e2e-test resourceVersion: "26893" uid: 4c8895c9-da1e-44e2-a614-0e9f1be9215d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T18:12:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:12:36Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c686a7a56847a9 namespace: kserve-ci-e2e-test resourceVersion: "34070" uid: f8d1c0f1-a8ba-440d-8b13-af4581a5a2e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 9c10a7e1-e62e-4167-b3b4-41d398b449ba kind: Event lastTimestamp: "2026-07-28T18:12:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:12:36Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x.18c686a7a56d5544 namespace: kserve-ci-e2e-test resourceVersion: "34071" uid: 88b3b8b6-8bc1-4452-bed1-bcde8c877e9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cb1ee-predictor-8494ff7d66 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 234c7dd9-92eb-4fa8-af32-262165b168ed kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: 'Created pod: error-404-isvc-cb1ee-predictor-8494ff7d66-sv48x' metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:07Z" name: error-404-isvc-cb1ee-predictor-8494ff7d66.18c685b351201bf7 namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: b67109cf-4b6d-4410-8702-21bba329a0aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26473" uid: 4809baef-0c5b-4b1f-b751-eb1178d03497 kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: Scaled up replica set error-404-isvc-cb1ee-predictor-8494ff7d66 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:07Z" name: error-404-isvc-cb1ee-predictor.18c685b34e0a42d8 namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: 33479dfa-b384-4d1b-9e24-122fca5d7ecb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:55:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 6d92c112-08e0-4a94-80ca-107c076cc5ff kind: Event lastTimestamp: "2026-07-28T17:55:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:37Z" name: error-404-isvc-cb1ee-predictor.18c685b6ccc79d28 namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: a5223433-51e2-4148-8042-75e017b873cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:55:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 6d92c112-08e0-4a94-80ca-107c076cc5ff kind: Event lastTimestamp: "2026-07-28T17:55:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:37Z" name: error-404-isvc-cb1ee-predictor.18c685b6cd32dca0 namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: 7c5f1d06-b7c3-4d96-8547-c042b2acecd0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:55:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26695" uid: 6d92c112-08e0-4a94-80ca-107c076cc5ff kind: Event lastTimestamp: "2026-07-28T17:56:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:22Z" name: error-404-isvc-cb1ee-predictor.18c685bdcb0ff854 namespace: kserve-ci-e2e-test resourceVersion: "27292" uid: 8685d898-62b9-4315-be70-3e7ea1c440b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:55:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26695" uid: 6d92c112-08e0-4a94-80ca-107c076cc5ff kind: Event lastTimestamp: "2026-07-28T17:56:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:22Z" name: error-404-isvc-cb1ee-predictor.18c685bdcb8586cf namespace: kserve-ci-e2e-test resourceVersion: "27293" uid: fc02cf6b-9df8-4c01-982a-ee79d1d4b4d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 21e868da-cb13-4a1c-9fa1-967164765aaf kind: Event lastTimestamp: "2026-07-28T17:55:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:56Z" name: error-404-isvc-cb1ee.18c685b34e6131d5 namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: b48b7744-69fa-456a-9306-ff4a9f9d8875 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 21e868da-cb13-4a1c-9fa1-967164765aaf kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: 'Failed to update status for InferenceService "error-404-isvc-cb1ee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cb1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:07Z" name: error-404-isvc-cb1ee.18c685b34ee1fd7f namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 7adcdd10-1434-41bd-9d49-71b5945eec50 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 21e868da-cb13-4a1c-9fa1-967164765aaf kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cb1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:07Z" name: error-404-isvc-cb1ee.18c685b34ee25b8b namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: a76848d2-d727-41a8-ba04-46c7b9d5dda0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "26996" uid: 21e868da-cb13-4a1c-9fa1-967164765aaf kind: Event lastTimestamp: "2026-07-28T17:55:56Z" message: InferenceService [error-404-isvc-cb1ee] is Ready metadata: creationTimestamp: "2026-07-28T17:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:56Z" name: error-404-isvc-cb1ee.18c685be9ddac1be namespace: kserve-ci-e2e-test resourceVersion: "26997" uid: b5094ed3-20c7-40f5-bfbf-2cb1a0c6e2f6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:39.528353Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:39Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c68479776a8a04 namespace: kserve-ci-e2e-test resourceVersion: "14585" uid: 00d186d9-90d3-47ae-8ac8-140f086dfbd6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f0903-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:39Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c684798090c9d8 namespace: kserve-ci-e2e-test resourceVersion: "14607" uid: c4a02eea-595f-49b4-9316-1fe612e57c75 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:40Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c68479b56e4080 namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 8a19e54f-d8b2-4ca1-9947-7fff2439713d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:40Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c68479b74db41b namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: 61b2a56f-6ed6-4925-a100-091cb07df64b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" in 11.242s (11.242s including waiting). Image size: 1333940313 bytes.' metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:51Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6847c556cedc9 namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: b91b3124-76aa-42d6-8284-45ca5bfb01a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:51Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6847c58de9c23 namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: 3cdfc900-db36-4529-a542-baf1464cce67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:51Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6847c594c8a03 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 2cccb9b7-f823-41de-a4b6-6a770200bc8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:51Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6847c595aaced namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 22ba4307-41ea-4025-ae63-267f37e581f5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.167s (2.167s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:54Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6847cda88371f namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: cbb3b1da-baee-44e7-9497-1c5f338a84a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:54Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6847cddcf17a0 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 47768d01-e09a-4800-bec3-32c96cf7e1ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:32:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:54Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6847cde420e7e namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: 722f2cc4-dfe3-4843-8f75-9e35897a4a33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:32:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6847d31021baf namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: 05a82e85-8e1f-4995-9e80-6be286848493 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6848f607642f9 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: ab118ac0-c41e-453f-a6a8-33668bc1e855 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c6848f6076adfb namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 2733c3d0-be09-4353-b600-a9467d1916fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: f08dd03d-55a4-43ec-8a2a-6952e8d6a038 kind: Event lastTimestamp: "2026-07-28T17:34:16Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:16Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd.18c684900d40a567 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: acdbdbd8-04d6-4ec9-92c4-44a562a7a8d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f0903-predictor-5f4f5b79bb namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 7bb7dbf6-aab3-420c-9f3d-bfb325e31f2b kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'Created pod: error-404-isvc-f0903-predictor-5f4f5b79bb-nlqzd' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:39Z" name: error-404-isvc-f0903-predictor-5f4f5b79bb.18c6847976f01799 namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: 189edb79-598f-43f6-9314-a86f047db6f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 291a8ee3-2c46-45a3-9015-b3d78bb3e8ec kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: Scaled up replica set error-404-isvc-f0903-predictor-5f4f5b79bb from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:39Z" name: error-404-isvc-f0903-predictor.18c68479753aaf7e namespace: kserve-ci-e2e-test resourceVersion: "14575" uid: 04ec430f-5ddf-4f1f-ba8d-4d7af88ef87c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: 9593d7c6-03fc-4aee-8dc8-9e23f4eee070 kind: Event lastTimestamp: "2026-07-28T17:33:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:09Z" name: error-404-isvc-f0903-predictor.18c6847cf3cf7b8e namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: e76d5afb-a9df-4e7c-a991-827a4c89b191 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: 9593d7c6-03fc-4aee-8dc8-9e23f4eee070 kind: Event lastTimestamp: "2026-07-28T17:33:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:09Z" name: error-404-isvc-f0903-predictor.18c6847cf41cc49b namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: 9d6c1f56-52ae-493a-815e-31c0dd4ed736 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:33:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 9593d7c6-03fc-4aee-8dc8-9e23f4eee070 kind: Event lastTimestamp: "2026-07-28T17:34:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:09Z" name: error-404-isvc-f0903-predictor.18c68483f183d2c7 namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: df8d4c71-7fa7-4590-8a7e-0ee2c59bb641 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:33:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 9593d7c6-03fc-4aee-8dc8-9e23f4eee070 kind: Event lastTimestamp: "2026-07-28T17:34:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:09Z" name: error-404-isvc-f0903-predictor.18c68483f2042197 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: ffadefcf-42af-4aa9-b043-77bc7905a162 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0903 namespace: kserve-ci-e2e-test resourceVersion: "14555" uid: 79f108e9-eb47-4bd7-a0df-8343aafdba41 kind: Event lastTimestamp: "2026-07-28T17:33:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:41Z" name: error-404-isvc-f0903.18c6847975427b57 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 35084767-9f26-459f-a12a-282e64bf2a12 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0903 namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: 79f108e9-eb47-4bd7-a0df-8343aafdba41 kind: Event lastTimestamp: "2026-07-28T17:33:41Z" message: InferenceService [error-404-isvc-f0903] is Ready metadata: creationTimestamp: "2026-07-28T17:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:41Z" name: error-404-isvc-f0903.18c68487ec7f076c namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 4d3cb6f9-00ed-4673-b98f-caa8ab478dfd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:53:53.742559Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25306" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 to ip-10-0-141-165.ec2.internal metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:53:53Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a224805c3c namespace: kserve-ci-e2e-test resourceVersion: "25310" uid: b779f8d4-a4b4-40c8-81b7-b9800a1b55ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f7704-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:53Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a231301978 namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: 57a8131d-ffd7-4154-8595-92852d98ac7e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:53:54Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a2628c201e namespace: kserve-ci-e2e-test resourceVersion: "25362" uid: cf42ee2f-ecce-4349-89a1-4a59155c6514 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a2643cf543 namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 87b8ff92-37e6-4c89-8455-b45deba74bac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a2685c4196 namespace: kserve-ci-e2e-test resourceVersion: "25365" uid: 68db25e5-2b02-44a0-976a-12c642f07392 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a268dc2260 namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: 2cf0adc4-35b5-451e-a9d6-d95b8b07c070 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a268e77e19 namespace: kserve-ci-e2e-test resourceVersion: "25367" uid: 5fb71afb-69a3-45e8-818a-1c18ecbdcf04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a26c9d6d5f namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: def28c45-1a4a-4377-859f-684e9f9180bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a26d188952 namespace: kserve-ci-e2e-test resourceVersion: "25370" uid: 5a96f715-cdc3-4db7-834f-5df8b501e66d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:54:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:54:32Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685a2cc1ee758 namespace: kserve-ci-e2e-test resourceVersion: "25902" uid: 197639b6-aaae-4466-82ee-3026cf57a451 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:55:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:04Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685b28b1f43b5 namespace: kserve-ci-e2e-test resourceVersion: "26335" uid: 2a430ffc-8448-4f1d-bd3f-6c924cbc853a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:55:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:04Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685b28b1fd4ba namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: f80bd075-f111-4d36-8b51-7a5c4fe86972 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 8e664fad-d9a9-4cb8-914e-aa23c663d7aa kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: 'Readiness probe failed: Get "https://10.134.0.45:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4.18c685b390037bb1 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: fb6946ea-ed50-49ae-a31f-7b8355b72a23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-165.ec2.internal source: component: kubelet host: ip-10-0-141-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f7704-predictor-5b75c47dd6 namespace: kserve-ci-e2e-test resourceVersion: "25296" uid: e45715f4-4e5a-4387-a687-de7f8c1d9ce9 kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: 'Created pod: error-404-isvc-f7704-predictor-5b75c47dd6-jxdm4' metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:53:53Z" name: error-404-isvc-f7704-predictor-5b75c47dd6.18c685a223e317b5 namespace: kserve-ci-e2e-test resourceVersion: "25307" uid: e536c93c-02e3-4b6f-b629-9422a1aabe38 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25292" uid: 8fb5a9bc-f2a5-4a2f-b8f2-0841a994b07e kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: Scaled up replica set error-404-isvc-f7704-predictor-5b75c47dd6 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:53:53Z" name: error-404-isvc-f7704-predictor.18c685a221b4da26 namespace: kserve-ci-e2e-test resourceVersion: "25299" uid: 8c2d14d5-c4f2-4d26-8ab4-8c5c5e515038 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: b0103e40-c726-4988-b063-31b105e2eebb kind: Event lastTimestamp: "2026-07-28T17:54:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:38Z" name: error-404-isvc-f7704-predictor.18c685a5a035d92f namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 63569578-9818-4670-a340-dffc766f0ade reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: b0103e40-c726-4988-b063-31b105e2eebb kind: Event lastTimestamp: "2026-07-28T17:54:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:38Z" name: error-404-isvc-f7704-predictor.18c685a5a0acb058 namespace: kserve-ci-e2e-test resourceVersion: "26009" uid: 84e8b622-493c-479d-aa3a-f0c1b5ee17af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25497" uid: b0103e40-c726-4988-b063-31b105e2eebb kind: Event lastTimestamp: "2026-07-28T17:54:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:53Z" name: error-404-isvc-f7704-predictor.18c685b01cee0ef5 namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: b1153bcd-e745-4e76-a057-e1a9518b80ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25497" uid: b0103e40-c726-4988-b063-31b105e2eebb kind: Event lastTimestamp: "2026-07-28T17:54:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:53Z" name: error-404-isvc-f7704-predictor.18c685b01d616313 namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: df621352-a136-4382-afa8-617a52cd3cd8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7704 namespace: kserve-ci-e2e-test resourceVersion: "25277" uid: 83e89f36-d61b-4637-802f-ad21c97edaca kind: Event lastTimestamp: "2026-07-28T17:54:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:54:42Z" name: error-404-isvc-f7704.18c685a2220617ff namespace: kserve-ci-e2e-test resourceVersion: "26053" uid: 7c07e7ca-bd1b-4217-bfd6-8a65aed485d3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7704 namespace: kserve-ci-e2e-test resourceVersion: "25277" uid: 83e89f36-d61b-4637-802f-ad21c97edaca kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: 'Failed to update status for InferenceService "error-404-isvc-f7704": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7704": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:53:53Z" name: error-404-isvc-f7704.18c685a222b49377 namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 889059c2-33cd-47c8-a2a0-c60c13b4401d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7704 namespace: kserve-ci-e2e-test resourceVersion: "25277" uid: 83e89f36-d61b-4637-802f-ad21c97edaca kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7704": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:53:53Z" name: error-404-isvc-f7704.18c685a222b4d795 namespace: kserve-ci-e2e-test resourceVersion: "25305" uid: d894cd55-9935-461f-a889-42350511e606 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7704 namespace: kserve-ci-e2e-test resourceVersion: "26051" uid: 83e89f36-d61b-4637-802f-ad21c97edaca kind: Event lastTimestamp: "2026-07-28T17:54:42Z" message: InferenceService [error-404-isvc-f7704] is Ready metadata: creationTimestamp: "2026-07-28T17:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:54:42Z" name: error-404-isvc-f7704.18c685ad8831665c namespace: kserve-ci-e2e-test resourceVersion: "26052" uid: 26b54e55-7246-404c-b632-e5661e833005 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:39.317041Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg to ip-10-0-142-249.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:39Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c684796ad22166 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: b7cade1e-d3ae-4164-8abe-f07b62c545ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:39Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c684797acd4633 namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: e4c151c6-f5cb-464d-b8bd-96a8f26bdc8f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:40Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c68479a8ab940a namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: 4f7e1df6-612d-41b9-b9b4-11de32a2f019 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:40Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c68479aa3f5c62 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 569db830-7b0e-489f-9a73-44466c99007b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:40Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c68479ad0fd8f7 namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 20856be1-d0e4-41b6-b0f2-405e9dcf02c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:40Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c68479ad94d622 namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: 5642e3c2-df32-4832-8ae9-6c9b97326806 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:32:44Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:32:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:44Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c6847a9e6a9589 namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: 92b62982-9c40-4710-858e-b7868ce1e478 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:33:12Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" in 28.204s (28.204s including waiting). Image size: 1555722147 bytes.' metadata: creationTimestamp: "2026-07-28T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:12Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c684812f8a1558 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 4fbf30a0-3e38-48d5-9e5a-2889c502c726 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:33:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:12Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c6848136712365 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 9e029453-1d55-466b-846f-3b8a55b8260a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:33:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:12Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c6848137a1b87c namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: a03622be-561c-49b9-9bfb-51d043832d9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:33:12Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:12Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c6848137b39227 namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 54ca6d6f-d7ee-48a5-bf6a-61df88bbca67 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:33:15Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.008s (3.008s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:15Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c68481eaffef3d namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: b3d25eb0-ef4d-4cf6-a61b-e83d9481b2d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:33:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:15Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c68481f077dc0d namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: 7a1378b9-79d1-416c-94da-cbdd5252442c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:33:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:15Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c68481f0f0a33e namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: fd84cbb3-ce52-448f-b8e3-056016dd869f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:34:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:28Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c684825bdbf71c namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 87d30464-0eb5-4245-9350-855c0f9b1d3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c6849f993ca681 namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 49626e08-bb8d-4a7a-ad97-e1ff916bb199 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c6849f993d30d0 namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 2161e553-e440-4b90-968c-23ba0dfc3d98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: d5f1542d-ab00-4a28-9f7b-323565f72c04 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg.18c6849fb24c9d73 namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 0e54b6ac-76f1-420b-931b-80f714a9d9d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-84c58bd689 namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 31e26081-2dad-4470-82fe-b7533a231c5c kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-84c58bd689-6fbfg' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:39Z" name: isvc-sklearn-graph-1-predictor-84c58bd689.18c684796a46bb14 namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: a31bab53-57b1-4bb5-9936-6d02d4bc415f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 1ec30d4f-216c-458d-9032-455c71ff8aa5 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-84c58bd689 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:39Z" name: isvc-sklearn-graph-1-predictor.18c684796539b719 namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: 0060ca85-469c-44e7-8919-4a6f2bbe1f93 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 314490c7-d491-4625-b64a-7371e3e91a86 kind: Event lastTimestamp: "2026-07-28T17:33:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:54Z" name: isvc-sklearn-graph-1-predictor.18c6847ce3b7b231 namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: c1cdf96f-2da5-4574-aee2-0d6c27593737 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 314490c7-d491-4625-b64a-7371e3e91a86 kind: Event lastTimestamp: "2026-07-28T17:33:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:54Z" name: isvc-sklearn-graph-1-predictor.18c6847ce40516a9 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 1735f2c8-ec19-4dba-b21a-86de2e33bfdc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:34:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 314490c7-d491-4625-b64a-7371e3e91a86 kind: Event lastTimestamp: "2026-07-28T17:35:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:09Z" name: isvc-sklearn-graph-1-predictor.18c6848e5e0a0aaf namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: ce909aca-ef78-4506-b2f2-8f5f4b8883d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:34:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 314490c7-d491-4625-b64a-7371e3e91a86 kind: Event lastTimestamp: "2026-07-28T17:35:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:09Z" name: isvc-sklearn-graph-1-predictor.18c6848e5e7aec11 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 20eadbf3-2172-4988-afd1-8ee0538876d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 96ea8638-30d6-4010-b9bc-7090bcb81c01 kind: Event lastTimestamp: "2026-07-28T17:34:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:39Z" name: isvc-sklearn-graph-1.18c6847965826ef4 namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 2a805be6-3164-4bae-ad87-814bdb876064 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 96ea8638-30d6-4010-b9bc-7090bcb81c01 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:39Z" name: isvc-sklearn-graph-1.18c684796766e72b namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 98b6a827-cce4-4ab2-9b7e-3ae3dea0ae7a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 96ea8638-30d6-4010-b9bc-7090bcb81c01 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:39Z" name: isvc-sklearn-graph-1.18c6847967673f20 namespace: kserve-ci-e2e-test resourceVersion: "14524" uid: 938e986a-343e-4063-a715-56d2bd679bd3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: 96ea8638-30d6-4010-b9bc-7090bcb81c01 kind: Event lastTimestamp: "2026-07-28T17:34:38Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T17:34:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:38Z" name: isvc-sklearn-graph-1.18c68495484afdc5 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 924bdf36-ba2d-4a7d-8dd3-aa9a0cde7b45 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:39.716057Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14611" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl to ip-10-0-142-249.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:39Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c68479829aba3e namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 0d8fa7a5-4cb0-495c-8564-9e33c4239ecd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:40Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c684799c8fbc81 namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: 33fb34d4-5983-4a9a-9ab1-338a3451e9bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:40Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c684799df2db0a namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: b0b856f9-f8ef-42c3-a65e-0542fbe4d1bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:40Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c68479a1381f0a namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: a5b133cc-b954-46a4-a2b0-be315a9f9be7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:40Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c68479a1b03f9c namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 55093c7b-90ff-4fcc-8b48-4c2d76de1b55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:32:44Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:32:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:44Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c6847a9e53926a namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: fe0ec6f7-b825-4f17-a562-947d2e47eb45 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:33:12Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" in 28.204s (28.204s including waiting). Image size: 1555722147 bytes.' metadata: creationTimestamp: "2026-07-28T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:12Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c684812f793d76 namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 51e57243-104b-441e-8241-baa2c25fa85a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:33:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:12Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c68481367168bf namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: d77ffedb-97d6-42f9-8540-3b256c384896 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:33:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:12Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c6848137a1bdae namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: a4fbc4a1-6e3e-4fbd-98ba-7122350d120a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:33:12Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:12Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c6848137b1cfde namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: 03ce44ad-a681-4fdb-bb30-76e991d0b5ed reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:33:15Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.005s (3.005s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:15Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c68481eadaa6d6 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: e09e3b68-3b15-493a-8850-fa29b2c62c27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:33:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:15Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c68481f0786744 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 3f3c82a6-34c8-4e8b-943f-100dee7be44c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:33:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:15Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c68481f0ea6204 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 7653c8e4-e3d4-4039-bfa2-58df97447a3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:34:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:22Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c684821fba2e38 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 79393c54-9fc7-4675-9a45-920bd7bf16f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c6849f950783aa namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: cf0fa621-6f4a-42b7-9b60-9ed099034e67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c6849f9507efe1 namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: cefd4b99-7341-4652-a9f0-f3a3ecc01fc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: cfe21f4b-5b00-46b1-8e15-b4659e23cc84 kind: Event lastTimestamp: "2026-07-28T17:35:27Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:27Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl.18c684a0a06c369b namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: c349580b-c3cc-427f-ae82-0180fd48263c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-5bc9545b47 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: ef9c6912-aa0f-4698-b262-c1ae3292c201 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-5bc9545b47-6bbpl' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:39Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47.18c684798226d9ce namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: 0f906e2f-a46f-49a9-880c-9de59e5b36c2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: b8f6f1bd-b452-4607-985e-2846436a53c3 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-5bc9545b47 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:39Z" name: isvc-sklearn-graph-2-predictor.18c684797e58b9ef namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: de01a612-e78f-42a3-be0d-b7a24815d80c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 2ef4e272-9c03-4f4d-9cb1-b1ad27fb63dd kind: Event lastTimestamp: "2026-07-28T17:33:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:54Z" name: isvc-sklearn-graph-2-predictor.18c6847cfd3b4108 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: 8afcd2c0-86af-441a-a23d-5ce5adf64ed1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 2ef4e272-9c03-4f4d-9cb1-b1ad27fb63dd kind: Event lastTimestamp: "2026-07-28T17:33:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:54Z" name: isvc-sklearn-graph-2-predictor.18c6847cfddd2015 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: c0d5fa73-1468-47bf-b01c-554cc36b2fa7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:33:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 2ef4e272-9c03-4f4d-9cb1-b1ad27fb63dd kind: Event lastTimestamp: "2026-07-28T17:34:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:54Z" name: isvc-sklearn-graph-2-predictor.18c684877a9df2ae namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: eb418565-0f2f-4f89-a2b4-d5397143ef38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:33:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 2ef4e272-9c03-4f4d-9cb1-b1ad27fb63dd kind: Event lastTimestamp: "2026-07-28T17:34:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:54Z" name: isvc-sklearn-graph-2-predictor.18c684877b06c5a8 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 4a10e984-457b-471b-9b38-5c0c528c920d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: e8c7269b-a4cc-43d8-a1ee-1ca5ef2609b0 kind: Event lastTimestamp: "2026-07-28T17:34:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:32Z" name: isvc-sklearn-graph-2.18c684797e59d1fc namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: a6387708-144e-41a0-8791-cb3abc53aa43 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: e8c7269b-a4cc-43d8-a1ee-1ca5ef2609b0 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T17:34:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:31Z" name: isvc-sklearn-graph-2.18c68493a6c6422d namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: ba6d9fa6-00e5-4adc-8fc7-7037edbf58f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:39.439912Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-tbg9w to ip-10-0-142-249.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c68479722518ac namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 319120da-c260-41e9-abc4-74b80fa5cf35 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c6847980cfe019 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 9a1f5c0e-acb9-4542-9598-94e03b15a9dc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:32:41Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c68479e54d8dee namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: f3426281-1b91-4b60-9863-c94c170be808 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:32:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c68479e6c3e8f9 namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 62f7de8d-0f92-42af-a0bb-424c55fcf284 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:32:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c68479e98c015f namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: a8a886bf-daa1-47f2-a382-dde9b2bb581b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:32:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c68479ea11368b namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 29a4459f-8601-4589-894e-12cab08ade9d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:32:46Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T17:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:46Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c6847b1892c0a2 namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 7eaaedf4-bc86-4c40-97cb-27cd8fae916c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:33:13Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 26.78s (26.78s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T17:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c6848154cfc767 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 0018509f-0216-4bf5-afd9-33a0c1e45500 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:33:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c684815a98e985 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: e4c6f6b5-75e3-4907-a0ed-332462ad3c02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:33:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c684815b2acc00 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: cffcc756-46e3-48fe-b1ec-55db43796c33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:33:13Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c684815b3dc05e namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 73d6856f-10cf-4be6-8328-37b9fc2b4325 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:33:15Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.435s (2.435s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c68481ec64e9f3 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: c644e7fe-ae4d-456e-9d99-bf22ff0b8882 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:33:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c68481f07aa3d3 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 478aa7a7-1085-4ba7-9a55-6ddea878e290 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:33:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c68481f0ea0bc4 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 847995c8-baf5-420e-8df2-ceba74f9885a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:34:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c684821fd4fe72 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 30abebea-d13c-4f33-8d02-eac6c1cd285a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c6849f9befde5f namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: 8b88baf9-792c-4762-bd1e-fc94c617dbc9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 77a92b8f-69fe-4718-bfeb-bf97aa539127 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: isvc-xgboost-graph-predictor-555fc69d56-tbg9w.18c6849f9bf09b60 namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: 810c82c2-7e0f-40b6-9ba9-f7c343fd3b67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-249.ec2.internal source: component: kubelet host: ip-10-0-142-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: 6cee4ec9-34ee-48be-896f-8e9336750e84 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-tbg9w' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:39Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c6847971ba5ff7 namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: fcd68d94-26a9-492c-9190-dfc932c2038b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: e8ae6230-43c9-43be-a8f9-65822a70152c kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:39Z" name: isvc-xgboost-graph-predictor.18c684796ee6de59 namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: fd4bf33f-ab0b-40f3-afee-ae344e57a44e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: fcffd975-ede0-4677-afd2-04a1c8d130ae kind: Event lastTimestamp: "2026-07-28T17:33:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:24Z" name: isvc-xgboost-graph-predictor.18c6847ced5aa937 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 0746c1e9-f474-411c-9fee-95e4bc1cb120 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: fcffd975-ede0-4677-afd2-04a1c8d130ae kind: Event lastTimestamp: "2026-07-28T17:33:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:24Z" name: isvc-xgboost-graph-predictor.18c6847cedc4ed52 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: d4f8c29c-02e3-4121-9597-1e8bcc7aad57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: fcffd975-ede0-4677-afd2-04a1c8d130ae kind: Event lastTimestamp: "2026-07-28T17:34:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:54Z" name: isvc-xgboost-graph-predictor.18c684876a0d5d0c namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 944daa6f-47f1-449a-a187-9be3a1b0d3bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: fcffd975-ede0-4677-afd2-04a1c8d130ae kind: Event lastTimestamp: "2026-07-28T17:34:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:54Z" name: isvc-xgboost-graph-predictor.18c684876a77bd6a namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: d3f4d2ba-9d4a-4cda-89cd-2cd901e761e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: 04441243-2ac8-40ae-824b-d146873e07ad kind: Event lastTimestamp: "2026-07-28T17:34:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:22Z" name: isvc-xgboost-graph.18c684796ede7270 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 8b1cbbde-2159-4d25-b52f-3156156f3239 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 04441243-2ac8-40ae-824b-d146873e07ad kind: Event lastTimestamp: "2026-07-28T17:34:22Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:22Z" name: isvc-xgboost-graph.18c684918427004e namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 73daf142-8c10-4f36-9307-57a1cd839c39 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:35:09.016972Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64698579b9-v2bps namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: b8c677af-aff7-468b-a3fa-c9718b98b0b6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-64698579b9-v2bps to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:35:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:35:09Z" name: model-chainer-64698579b9-v2bps.18c6849c45a1cc3f namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: a65f4401-b70b-4305-a3bd-0020fafb3732 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64698579b9-v2bps namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: b8c677af-aff7-468b-a3fa-c9718b98b0b6 kind: Event lastTimestamp: "2026-07-28T17:35:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:09Z" name: model-chainer-64698579b9-v2bps.18c6849c51a66944 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 490fd71e-499d-4eba-b312-2a6d897b0561 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:10Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64698579b9-v2bps namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: b8c677af-aff7-468b-a3fa-c9718b98b0b6 kind: Event lastTimestamp: "2026-07-28T17:35:10Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:35:10Z" name: model-chainer-64698579b9-v2bps.18c6849c8221afe9 namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 89a79d77-d623-47cd-b935-bc87a27a0670 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-v2bps namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: b8c677af-aff7-468b-a3fa-c9718b98b0b6 kind: Event lastTimestamp: "2026-07-28T17:35:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:10Z" name: model-chainer-64698579b9-v2bps.18c6849c839a977f namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: df39eeac-122f-4796-a688-476be350a726 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-v2bps namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: b8c677af-aff7-468b-a3fa-c9718b98b0b6 kind: Event lastTimestamp: "2026-07-28T17:35:10Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T17:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:10Z" name: model-chainer-64698579b9-v2bps.18c6849c86521741 namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: 84483dc2-7878-466d-a3a1-67d0a287c51a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-v2bps namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: b8c677af-aff7-468b-a3fa-c9718b98b0b6 kind: Event lastTimestamp: "2026-07-28T17:35:10Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T17:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:10Z" name: model-chainer-64698579b9-v2bps.18c6849c86c25c52 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: 39e8856e-ddd4-4bc8-a941-ef010b00b23e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-v2bps namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: b8c677af-aff7-468b-a3fa-c9718b98b0b6 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: model-chainer-64698579b9-v2bps.18c6849f8d57cd5a namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: f77bdadb-a413-4c99-a864-2a39313dabff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-v2bps namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: b8c677af-aff7-468b-a3fa-c9718b98b0b6 kind: Event lastTimestamp: "2026-07-28T17:35:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:51Z" name: model-chainer-64698579b9-v2bps.18c684a06361c228 namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: cc0abb03-eb72-4e60-ab92-6c86e93a1a9b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-64698579b9 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 6fcf9423-c592-46c9-8c7d-ba5212866d3f kind: Event lastTimestamp: "2026-07-28T17:35:09Z" message: 'Created pod: model-chainer-64698579b9-v2bps' metadata: creationTimestamp: "2026-07-28T17:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:09Z" name: model-chainer-64698579b9.18c6849c44c497e7 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 23ecba8a-b74d-408d-b4e5-289e46123649 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: a57f7807-a933-4df0-9c67-de27415af896 kind: Event lastTimestamp: "2026-07-28T17:35:08Z" message: Scaled up replica set model-chainer-64698579b9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:08Z" name: model-chainer.18c6849c43280864 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: b786be62-f333-49b8-8d2e-ccf976dfdb34 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: a67505f3-9787-4e78-820d-7463071c201d kind: Event lastTimestamp: "2026-07-28T17:35:09Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T17:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:09Z" name: model-chainer.18c6849c4affa4d0 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: 77dec049-8ce3-4182-b339-eb17bb87612c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: a67505f3-9787-4e78-820d-7463071c201d kind: Event lastTimestamp: "2026-07-28T17:35:09Z" message: 'Failed to update status for InferenceGraph "model-chainer": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:09Z" name: model-chainer.18c6849c4dc1192c namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: c5347e4c-702a-4957-9825-ab746aa28f27 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: a67505f3-9787-4e78-820d-7463071c201d kind: Event lastTimestamp: "2026-07-28T17:35:09Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:09Z" name: model-chainer.18c6849c4dc16271 namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: de1a126c-ff6b-4562-9633-1f685d5e8806 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:45:38.390352Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-61a23-7dfbd4b7cf-ll569 namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: 989befef-f869-4ed5-ac37-9e447c15b756 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-61a23-7dfbd4b7cf-ll569 to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:45:38Z" name: sequence-graph-61a23-7dfbd4b7cf-ll569.18c6852ecf3575f6 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: e2a4dc5c-0838-4570-9ad8-ebc7ef2e3ecc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-61a23-7dfbd4b7cf-ll569 namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: 989befef-f869-4ed5-ac37-9e447c15b756 kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:45:38Z" name: sequence-graph-61a23-7dfbd4b7cf-ll569.18c6852ee8b5b799 namespace: kserve-ci-e2e-test resourceVersion: "21573" uid: 58475e85-5cfe-458d-805c-e0a4c9b6f8f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-61a23} kind: Pod name: sequence-graph-61a23-7dfbd4b7cf-ll569 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 989befef-f869-4ed5-ac37-9e447c15b756 kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:38Z" name: sequence-graph-61a23-7dfbd4b7cf-ll569.18c6852eea3e805b namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: e05932fe-9696-4a21-84de-d853430ad8e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-61a23} kind: Pod name: sequence-graph-61a23-7dfbd4b7cf-ll569 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 989befef-f869-4ed5-ac37-9e447c15b756 kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: 'Created container: sequence-graph-61a23' metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:38Z" name: sequence-graph-61a23-7dfbd4b7cf-ll569.18c6852eed287af9 namespace: kserve-ci-e2e-test resourceVersion: "21579" uid: b92efeb1-cdc7-40c2-833e-7a2f662defc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-61a23} kind: Pod name: sequence-graph-61a23-7dfbd4b7cf-ll569 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 989befef-f869-4ed5-ac37-9e447c15b756 kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: Started container sequence-graph-61a23 metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:38Z" name: sequence-graph-61a23-7dfbd4b7cf-ll569.18c6852eed9c9811 namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: 98ebef00-8567-4e39-bd45-acebaaeec98b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-61a23} kind: Pod name: sequence-graph-61a23-7dfbd4b7cf-ll569 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 989befef-f869-4ed5-ac37-9e447c15b756 kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: Stopping container sequence-graph-61a23 metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:52Z" name: sequence-graph-61a23-7dfbd4b7cf-ll569.18c685320db01d60 namespace: kserve-ci-e2e-test resourceVersion: "21717" uid: 43e2c227-a3ad-4741-91a9-97c5afcfd656 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:45:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-61a23} kind: Pod name: sequence-graph-61a23-7dfbd4b7cf-ll569 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 989befef-f869-4ed5-ac37-9e447c15b756 kind: Event lastTimestamp: "2026-07-28T17:46:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:45:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:46:20Z" name: sequence-graph-61a23-7dfbd4b7cf-ll569.18c68532e337bf6a namespace: kserve-ci-e2e-test resourceVersion: "22076" uid: 1af2f4b1-345e-4eb9-adf4-5be170d91fce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-61a23-7dfbd4b7cf namespace: kserve-ci-e2e-test resourceVersion: "21520" uid: f5f092d1-40fa-468f-bd99-01baa24dde3f kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: 'Created pod: sequence-graph-61a23-7dfbd4b7cf-ll569' metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:38Z" name: sequence-graph-61a23-7dfbd4b7cf.18c6852ecea1cc8d namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: a6e7a817-e77d-4661-a848-8adeed3ba829 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-61a23 namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: 2c93a987-20bf-4412-9505-5a046aa3530c kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: Scaled up replica set sequence-graph-61a23-7dfbd4b7cf from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:38Z" name: sequence-graph-61a23.18c6852eca9e7bb6 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 74b5ac51-3f4b-4fdb-883b-8e36c59242d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-61a23 namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: 56dd5830-5f7d-4821-b053-561583246d4f kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: InferenceGraph [sequence-graph-61a23] is Ready metadata: creationTimestamp: "2026-07-28T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:38Z" name: sequence-graph-61a23.18c6852ed3ff422f namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: 049e08f3-3c34-47fd-9b8b-5c5f917c62cd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:46:52.573271Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-787a1-77c7b956dc-cpngn namespace: kserve-ci-e2e-test resourceVersion: "22335" uid: 811f7ddb-91b4-4102-9ee5-ecb2a04ab80f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-787a1-77c7b956dc-cpngn to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:46:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:46:52Z" name: sequence-graph-787a1-77c7b956dc-cpngn.18c6854014dafb0b namespace: kserve-ci-e2e-test resourceVersion: "22340" uid: 934bb73b-33b1-490a-b317-ea592861b732 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:53Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-787a1-77c7b956dc-cpngn namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: 811f7ddb-91b4-4102-9ee5-ecb2a04ab80f kind: Event lastTimestamp: "2026-07-28T17:46:53Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:46:53Z" name: sequence-graph-787a1-77c7b956dc-cpngn.18c685402eb1ddb9 namespace: kserve-ci-e2e-test resourceVersion: "22357" uid: 96219a12-61d0-4011-a710-9086a82beef9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-787a1} kind: Pod name: sequence-graph-787a1-77c7b956dc-cpngn namespace: kserve-ci-e2e-test resourceVersion: "22338" uid: 811f7ddb-91b4-4102-9ee5-ecb2a04ab80f kind: Event lastTimestamp: "2026-07-28T17:46:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:46:53Z" name: sequence-graph-787a1-77c7b956dc-cpngn.18c68540302917a7 namespace: kserve-ci-e2e-test resourceVersion: "22359" uid: 41d57cd0-6886-4c03-a639-e92f435d37e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-787a1} kind: Pod name: sequence-graph-787a1-77c7b956dc-cpngn namespace: kserve-ci-e2e-test resourceVersion: "22338" uid: 811f7ddb-91b4-4102-9ee5-ecb2a04ab80f kind: Event lastTimestamp: "2026-07-28T17:46:53Z" message: 'Created container: sequence-graph-787a1' metadata: creationTimestamp: "2026-07-28T17:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:46:53Z" name: sequence-graph-787a1-77c7b956dc-cpngn.18c6854033d7bbba namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: 52d402ae-a31b-417e-9caf-442e400fa1ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-787a1} kind: Pod name: sequence-graph-787a1-77c7b956dc-cpngn namespace: kserve-ci-e2e-test resourceVersion: "22338" uid: 811f7ddb-91b4-4102-9ee5-ecb2a04ab80f kind: Event lastTimestamp: "2026-07-28T17:46:53Z" message: Started container sequence-graph-787a1 metadata: creationTimestamp: "2026-07-28T17:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:46:53Z" name: sequence-graph-787a1-77c7b956dc-cpngn.18c68540349639fb namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: f8aad561-aed6-4c77-818a-0145ac8cb9dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-787a1} kind: Pod name: sequence-graph-787a1-77c7b956dc-cpngn namespace: kserve-ci-e2e-test resourceVersion: "22338" uid: 811f7ddb-91b4-4102-9ee5-ecb2a04ab80f kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: Stopping container sequence-graph-787a1 metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:07Z" name: sequence-graph-787a1-77c7b956dc-cpngn.18c685b33db401eb namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 79374a51-27af-4de4-b3dc-33f445b1820a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-787a1} kind: Pod name: sequence-graph-787a1-77c7b956dc-cpngn namespace: kserve-ci-e2e-test resourceVersion: "22338" uid: 811f7ddb-91b4-4102-9ee5-ecb2a04ab80f kind: Event lastTimestamp: "2026-07-28T17:55:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:34Z" name: sequence-graph-787a1-77c7b956dc-cpngn.18c685b3b4ed3241 namespace: kserve-ci-e2e-test resourceVersion: "26797" uid: a0cbe5c6-3297-4856-acc1-edd695c8561a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-787a1-77c7b956dc namespace: kserve-ci-e2e-test resourceVersion: "22331" uid: 26079ac2-9a66-42b0-8ff6-b88dd915b23a kind: Event lastTimestamp: "2026-07-28T17:46:52Z" message: 'Created pod: sequence-graph-787a1-77c7b956dc-cpngn' metadata: creationTimestamp: "2026-07-28T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:46:52Z" name: sequence-graph-787a1-77c7b956dc.18c685401442403f namespace: kserve-ci-e2e-test resourceVersion: "22337" uid: 643b051f-c8c7-4cb6-b8bc-c4b983061224 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-787a1 namespace: kserve-ci-e2e-test resourceVersion: "22326" uid: be7fd4ce-8b2f-4d1b-8651-388362ccc1d9 kind: Event lastTimestamp: "2026-07-28T17:46:52Z" message: Scaled up replica set sequence-graph-787a1-77c7b956dc from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:46:52Z" name: sequence-graph-787a1.18c685401312911b namespace: kserve-ci-e2e-test resourceVersion: "22332" uid: 51ffa6e4-6ffe-47e1-8771-8cfa088cc775 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-787a1 namespace: kserve-ci-e2e-test resourceVersion: "22347" uid: 28db0079-44b8-415f-b7be-c4dc5f0125f5 kind: Event lastTimestamp: "2026-07-28T17:46:52Z" message: InferenceGraph [sequence-graph-787a1] is Ready metadata: creationTimestamp: "2026-07-28T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:46:52Z" name: sequence-graph-787a1.18c68540199d30db namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: 88d62adc-1aec-4cb4-893f-15c9e5a55fcc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-787a1 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: b4082b67-9bca-445d-a154-52a73678151c kind: Event lastTimestamp: "2026-07-28T17:47:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:37Z" name: sequence-graph-787a1.18c6854391aff57c namespace: kserve-ci-e2e-test resourceVersion: "22681" uid: 582f882e-337d-4f70-8b47-683b80a3be91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-787a1 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: b4082b67-9bca-445d-a154-52a73678151c kind: Event lastTimestamp: "2026-07-28T17:47:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:37Z" name: sequence-graph-787a1.18c685439269715e namespace: kserve-ci-e2e-test resourceVersion: "22682" uid: 0e4e0b29-650b-45f3-9515-8dc7345fbddd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:36:23.351015Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b1dfe-bf8f99978-2vj8v namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: bd5242f7-86f5-4cde-9584-8c75ce5dfe4b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-b1dfe-bf8f99978-2vj8v to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:36:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:36:23Z" name: sequence-graph-b1dfe-bf8f99978-2vj8v.18c684ad94495a2f namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: dd7f0286-fd82-4870-b2ec-a375cb830f59 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:23Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b1dfe-bf8f99978-2vj8v namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: bd5242f7-86f5-4cde-9584-8c75ce5dfe4b kind: Event lastTimestamp: "2026-07-28T17:36:23Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:36:23Z" name: sequence-graph-b1dfe-bf8f99978-2vj8v.18c684adad4d3714 namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: 7164174b-6a56-4f46-987f-60620ca2319c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b1dfe} kind: Pod name: sequence-graph-b1dfe-bf8f99978-2vj8v namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: bd5242f7-86f5-4cde-9584-8c75ce5dfe4b kind: Event lastTimestamp: "2026-07-28T17:36:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:36:23Z" name: sequence-graph-b1dfe-bf8f99978-2vj8v.18c684adae8efcd6 namespace: kserve-ci-e2e-test resourceVersion: "17056" uid: 0009a142-c727-46c9-a1cd-cd2740449094 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b1dfe} kind: Pod name: sequence-graph-b1dfe-bf8f99978-2vj8v namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: bd5242f7-86f5-4cde-9584-8c75ce5dfe4b kind: Event lastTimestamp: "2026-07-28T17:36:23Z" message: 'Created container: sequence-graph-b1dfe' metadata: creationTimestamp: "2026-07-28T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:36:23Z" name: sequence-graph-b1dfe-bf8f99978-2vj8v.18c684adb1668b2c namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: ca1ce067-b54e-4978-bab9-19c12336b192 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b1dfe} kind: Pod name: sequence-graph-b1dfe-bf8f99978-2vj8v namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: bd5242f7-86f5-4cde-9584-8c75ce5dfe4b kind: Event lastTimestamp: "2026-07-28T17:36:23Z" message: Started container sequence-graph-b1dfe metadata: creationTimestamp: "2026-07-28T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:36:23Z" name: sequence-graph-b1dfe-bf8f99978-2vj8v.18c684adb21a9946 namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 4df4714a-81a8-43a8-b947-8fc4a03d7991 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b1dfe} kind: Pod name: sequence-graph-b1dfe-bf8f99978-2vj8v namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: bd5242f7-86f5-4cde-9584-8c75ce5dfe4b kind: Event lastTimestamp: "2026-07-28T17:44:37Z" message: Stopping container sequence-graph-b1dfe metadata: creationTimestamp: "2026-07-28T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:37Z" name: sequence-graph-b1dfe-bf8f99978-2vj8v.18c68520be29ca75 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: cb56c32a-648e-4e80-a90d-e351cbdd0a82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b1dfe} kind: Pod name: sequence-graph-b1dfe-bf8f99978-2vj8v namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: bd5242f7-86f5-4cde-9584-8c75ce5dfe4b kind: Event lastTimestamp: "2026-07-28T17:45:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:04Z" name: sequence-graph-b1dfe-bf8f99978-2vj8v.18c685213421524c namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 3621b06a-0d84-4640-a3c7-047572c50d99 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-b1dfe-bf8f99978 namespace: kserve-ci-e2e-test resourceVersion: "17027" uid: 8dd63fff-d407-45df-b457-f4c61ce89cb5 kind: Event lastTimestamp: "2026-07-28T17:36:23Z" message: 'Created pod: sequence-graph-b1dfe-bf8f99978-2vj8v' metadata: creationTimestamp: "2026-07-28T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:23Z" name: sequence-graph-b1dfe-bf8f99978.18c684ad93cf08c1 namespace: kserve-ci-e2e-test resourceVersion: "17032" uid: 7d8a7c2f-c52f-4e74-ad9b-63e77da297e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: 2ff0001c-1bd8-448c-a3ed-7c6e75a17db4 kind: Event lastTimestamp: "2026-07-28T17:36:23Z" message: Scaled up replica set sequence-graph-b1dfe-bf8f99978 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:23Z" name: sequence-graph-b1dfe.18c684ad92e4043c namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 4b058319-bdbb-43e1-bd1b-8fd329537317 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 081dbd04-0995-4eed-a86d-d146bf927b25 kind: Event lastTimestamp: "2026-07-28T17:36:23Z" message: InferenceGraph [sequence-graph-b1dfe] is Ready metadata: creationTimestamp: "2026-07-28T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:36:23Z" name: sequence-graph-b1dfe.18c684ad98e81dc2 namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: b24dd0b0-c589-4c18-b7f9-6ce2a249474d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: 69369d58-45fe-4879-abd9-f3fe7f69de47 kind: Event lastTimestamp: "2026-07-28T17:36:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:38Z" name: sequence-graph-b1dfe.18c684b111820a7e namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: c7ee4a08-2f05-4d07-bba7-0398e7e12eec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: 69369d58-45fe-4879-abd9-f3fe7f69de47 kind: Event lastTimestamp: "2026-07-28T17:36:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:38Z" name: sequence-graph-b1dfe.18c684b111dc10ca namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 64e61a7f-1b9e-4bf1-9323-e13bc7432c7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "17163" uid: 69369d58-45fe-4879-abd9-f3fe7f69de47 kind: Event lastTimestamp: "2026-07-28T17:36:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:53Z" name: sequence-graph-b1dfe.18c684b4911b3bce namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: 5f5964d3-d83c-487e-a766-f1feb5b719a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "17163" uid: 69369d58-45fe-4879-abd9-f3fe7f69de47 kind: Event lastTimestamp: "2026-07-28T17:36:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:53Z" name: sequence-graph-b1dfe.18c684b49182509c namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 9d39893e-8068-4f71-a11e-9f2a45f5c6d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:56:04.063965Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7d999-849c5688bd-rrgg7 namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: a5cba631-1a98-41a8-b3c8-c9675bb989df kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-7d999-849c5688bd-rrgg7 to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999-849c5688bd-rrgg7.18c685c07c432869 namespace: kserve-ci-e2e-test resourceVersion: "27070" uid: 2cadc5ed-e539-495a-83f6-a928c092bcfe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:04Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7d999-849c5688bd-rrgg7 namespace: kserve-ci-e2e-test resourceVersion: "27074" uid: a5cba631-1a98-41a8-b3c8-c9675bb989df kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999-849c5688bd-rrgg7.18c685c096662317 namespace: kserve-ci-e2e-test resourceVersion: "27089" uid: 31989833-4c65-4b03-88e7-9116289698a1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d999} kind: Pod name: splitter-graph-7d999-849c5688bd-rrgg7 namespace: kserve-ci-e2e-test resourceVersion: "27066" uid: a5cba631-1a98-41a8-b3c8-c9675bb989df kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999-849c5688bd-rrgg7.18c685c097e013ad namespace: kserve-ci-e2e-test resourceVersion: "27092" uid: 8b9e5d29-4234-40b3-882f-c79cf37e988d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d999} kind: Pod name: splitter-graph-7d999-849c5688bd-rrgg7 namespace: kserve-ci-e2e-test resourceVersion: "27066" uid: a5cba631-1a98-41a8-b3c8-c9675bb989df kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: 'Created container: splitter-graph-7d999' metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999-849c5688bd-rrgg7.18c685c09ad41e22 namespace: kserve-ci-e2e-test resourceVersion: "27093" uid: a438ab2b-c921-4548-8fb8-7078f2c74ce7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d999} kind: Pod name: splitter-graph-7d999-849c5688bd-rrgg7 namespace: kserve-ci-e2e-test resourceVersion: "27066" uid: a5cba631-1a98-41a8-b3c8-c9675bb989df kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: Started container splitter-graph-7d999 metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999-849c5688bd-rrgg7.18c685c09b4fd888 namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: 8d8967f5-c55b-4e09-99d4-f93cb64fa610 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:04:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d999} kind: Pod name: splitter-graph-7d999-849c5688bd-rrgg7 namespace: kserve-ci-e2e-test resourceVersion: "27066" uid: a5cba631-1a98-41a8-b3c8-c9675bb989df kind: Event lastTimestamp: "2026-07-28T18:04:18Z" message: Stopping container splitter-graph-7d999 metadata: creationTimestamp: "2026-07-28T18:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:04:18Z" name: splitter-graph-7d999-849c5688bd-rrgg7.18c68633a5a94edb namespace: kserve-ci-e2e-test resourceVersion: "30588" uid: 67bbdba2-e898-47cc-a43a-cf60d4c0f05f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T18:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d999} kind: Pod name: splitter-graph-7d999-849c5688bd-rrgg7 namespace: kserve-ci-e2e-test resourceVersion: "27066" uid: a5cba631-1a98-41a8-b3c8-c9675bb989df kind: Event lastTimestamp: "2026-07-28T18:04:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T18:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:04:46Z" name: splitter-graph-7d999-849c5688bd-rrgg7.18c686343e142ea0 namespace: kserve-ci-e2e-test resourceVersion: "30823" uid: bba693b9-b1f3-44b2-a6ac-9570fbbc9b00 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-7d999-849c5688bd namespace: kserve-ci-e2e-test resourceVersion: "27059" uid: 84686b88-918b-4150-8fdb-8ac7cea919d3 kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: 'Created pod: splitter-graph-7d999-849c5688bd-rrgg7' metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999-849c5688bd.18c685c07ba3dc07 namespace: kserve-ci-e2e-test resourceVersion: "27065" uid: 964daebb-40f9-4d47-8c3a-c1056185811c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-7d999 namespace: kserve-ci-e2e-test resourceVersion: "27057" uid: f150a5b8-9d1f-40f2-80e0-e3647a4223a3 kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: Scaled up replica set splitter-graph-7d999-849c5688bd from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999.18c685c07ab992bb namespace: kserve-ci-e2e-test resourceVersion: "27061" uid: a2fac71d-7a56-4e1c-8c92-d367a92e5785 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-7d999 namespace: kserve-ci-e2e-test resourceVersion: "27077" uid: b4025d6e-70fd-4373-8d18-0c34f012fa9e kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: InferenceGraph [splitter-graph-7d999] is Ready metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999.18c685c081f82769 namespace: kserve-ci-e2e-test resourceVersion: "27079" uid: 505f3641-c539-4955-a2db-4e9dad5316f8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-7d999 namespace: kserve-ci-e2e-test resourceVersion: "27052" uid: b4025d6e-70fd-4373-8d18-0c34f012fa9e kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: 'Failed to update status for InferenceGraph "splitter-graph-7d999": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-7d999": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999.18c685c084a18728 namespace: kserve-ci-e2e-test resourceVersion: "27082" uid: 156e5374-015c-4edf-9f4d-235872a74019 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-7d999 namespace: kserve-ci-e2e-test resourceVersion: "27052" uid: b4025d6e-70fd-4373-8d18-0c34f012fa9e kind: Event lastTimestamp: "2026-07-28T17:56:04Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-7d999": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:56:04Z" name: splitter-graph-7d999.18c685c084a1d227 namespace: kserve-ci-e2e-test resourceVersion: "27084" uid: 04f690c6-8f83-43fd-8a89-ff01acc2c340 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7d999 namespace: kserve-ci-e2e-test resourceVersion: "27058" uid: 4ab546be-f8f5-435a-a66d-83c7634616cd kind: Event lastTimestamp: "2026-07-28T17:56:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:34Z" name: splitter-graph-7d999.18c685c3f9074b4c namespace: kserve-ci-e2e-test resourceVersion: "27379" uid: 224fa8b7-f969-44fb-adb1-dfeff98312d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7d999 namespace: kserve-ci-e2e-test resourceVersion: "27058" uid: 4ab546be-f8f5-435a-a66d-83c7634616cd kind: Event lastTimestamp: "2026-07-28T17:56:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:34Z" name: splitter-graph-7d999.18c685c3f95ea782 namespace: kserve-ci-e2e-test resourceVersion: "27380" uid: a5d7a9cf-30ed-4363-bbe4-bfd1c633bfa7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:54:53.710809Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f7704-758c568bf-l924l namespace: kserve-ci-e2e-test resourceVersion: "26141" uid: fcb68714-b150-4245-8506-40ff87015604 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-f7704-758c568bf-l924l to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:54:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:54:53Z" name: splitter-graph-f7704-758c568bf-l924l.18c685b01ae34436 namespace: kserve-ci-e2e-test resourceVersion: "26147" uid: 90afe47f-203e-47c5-902b-13d30729bb2d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:53Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f7704-758c568bf-l924l namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: fcb68714-b150-4245-8506-40ff87015604 kind: Event lastTimestamp: "2026-07-28T17:54:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-f7704-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:54:53Z" name: splitter-graph-f7704-758c568bf-l924l.18c685b0247684fb namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: ef8a0421-4d18-41c0-bbdb-d7aef2765923 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:54Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f7704-758c568bf-l924l namespace: kserve-ci-e2e-test resourceVersion: "26151" uid: fcb68714-b150-4245-8506-40ff87015604 kind: Event lastTimestamp: "2026-07-28T17:54:54Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:54:54Z" name: splitter-graph-f7704-758c568bf-l924l.18c685b058d622f8 namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 916ba70b-a2e2-4dfc-a7c9-f175a4ad6970 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7704} kind: Pod name: splitter-graph-f7704-758c568bf-l924l namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: fcb68714-b150-4245-8506-40ff87015604 kind: Event lastTimestamp: "2026-07-28T17:54:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:54:54Z" name: splitter-graph-f7704-758c568bf-l924l.18c685b05a62e30b namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 74ba6958-0910-4297-a9b7-d0e258aa40f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7704} kind: Pod name: splitter-graph-f7704-758c568bf-l924l namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: fcb68714-b150-4245-8506-40ff87015604 kind: Event lastTimestamp: "2026-07-28T17:54:54Z" message: 'Created container: splitter-graph-f7704' metadata: creationTimestamp: "2026-07-28T17:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:54:54Z" name: splitter-graph-f7704-758c568bf-l924l.18c685b05d475d12 namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: ac3feb17-af2e-4f54-939f-6500d60769d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7704} kind: Pod name: splitter-graph-f7704-758c568bf-l924l namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: fcb68714-b150-4245-8506-40ff87015604 kind: Event lastTimestamp: "2026-07-28T17:54:54Z" message: Started container splitter-graph-f7704 metadata: creationTimestamp: "2026-07-28T17:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:54:54Z" name: splitter-graph-f7704-758c568bf-l924l.18c685b05dc19052 namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: 72c49750-a3c8-412e-93d4-504445d5f485 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7704} kind: Pod name: splitter-graph-f7704-758c568bf-l924l namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: fcb68714-b150-4245-8506-40ff87015604 kind: Event lastTimestamp: "2026-07-28T17:55:03Z" message: Stopping container splitter-graph-f7704 metadata: creationTimestamp: "2026-07-28T17:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:03Z" name: splitter-graph-f7704-758c568bf-l924l.18c685b275693229 namespace: kserve-ci-e2e-test resourceVersion: "26261" uid: 6a8e2517-4f51-47ec-94db-4badb157aa91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7704} kind: Pod name: splitter-graph-f7704-758c568bf-l924l namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: fcb68714-b150-4245-8506-40ff87015604 kind: Event lastTimestamp: "2026-07-28T17:55:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:30Z" name: splitter-graph-f7704-758c568bf-l924l.18c685b2f3dcb58f namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: d7627510-f235-46e5-ad4f-076d455c6664 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-f7704-758c568bf namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 2953dafd-0bad-4224-a0df-70b044b5ede6 kind: Event lastTimestamp: "2026-07-28T17:54:53Z" message: 'Created pod: splitter-graph-f7704-758c568bf-l924l' metadata: creationTimestamp: "2026-07-28T17:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:53Z" name: splitter-graph-f7704-758c568bf.18c685b01a8f49a3 namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: 9acda8d0-9241-491e-862a-93d41bfcb346 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-f7704 namespace: kserve-ci-e2e-test resourceVersion: "26136" uid: 0bd3422b-8d2b-4266-81c5-a944fc2ca505 kind: Event lastTimestamp: "2026-07-28T17:54:53Z" message: Scaled up replica set splitter-graph-f7704-758c568bf from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:53Z" name: splitter-graph-f7704.18c685b019651674 namespace: kserve-ci-e2e-test resourceVersion: "26139" uid: e3be55dd-119e-4e8f-96b4-9acc380c7056 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-f7704 namespace: kserve-ci-e2e-test resourceVersion: "26156" uid: 3f87923b-d018-4f84-a876-7af4c586330e kind: Event lastTimestamp: "2026-07-28T17:54:53Z" message: InferenceGraph [splitter-graph-f7704] is Ready metadata: creationTimestamp: "2026-07-28T17:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:54:53Z" name: splitter-graph-f7704.18c685b01f24a0d1 namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: a1629feb-8d0d-4736-a9da-4a92f7c08d46 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:34:13.500778Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6848f589bfd0b namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: 381ad58d-157d-43a7-9ee3-fddea4aa916d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Add eth0 [10.133.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6848f719aac0e namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: c20cf7c5-83d8-4d99-aad6-9fb7ee3b5f6c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6848f72e66dea namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 381a0fa7-b7d2-4c90-9cb4-75a75c523798 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6848f75821502 namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: f2f05ce0-87fb-4eae-9ca3-b2893351d9e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6848f75f339c3 namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: 24357bba-088c-41a7-b89d-6983f237ba0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6848f75fd02fc namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: d3af34e9-7c25-4335-a30f-85e8c239c4d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:34:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:14Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6848f78c880d4 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 516118b7-efea-40c0-b61c-880ecd6b3290 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:34:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:14Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6848f79345e57 namespace: kserve-ci-e2e-test resourceVersion: "15679" uid: d0a79ca2-72e4-47c2-805d-f8a0104c1905 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:43:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:30Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6848f9457dea9 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: f237bc96-c805-4f8b-b478-de3df557f3bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:28Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6851086fea02a namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: a3c12740-3e50-4085-9321-f0b6652a179b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:28Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6851086ffdc36 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: 66a4f564-5a9c-4bf0-a6d7-e80453fe963e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 134b6f2e-e57e-44e6-b74c-e33cf4af9166 kind: Event lastTimestamp: "2026-07-28T17:43:30Z" message: 'Readiness probe failed: Get "https://10.133.0.16:8643/healthz": dial tcp 10.133.0.16:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:30Z" name: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4.18c6851108c006a5 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: debd6822-902b-4853-ba7a-b884f17faa6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-22e0e-predictor-dcf6f766c namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 33c4785a-ebab-4559-a695-244c5f030715 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: 'Created pod: success-200-isvc-22e0e-predictor-dcf6f766c-xcbz4' metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e-predictor-dcf6f766c.18c6848f5839db60 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: e21fd445-0fbf-4b89-9233-368081b0d3f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: c2752bee-4cef-43fc-a7de-62b9860d4479 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Scaled up replica set success-200-isvc-22e0e-predictor-dcf6f766c from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e-predictor.18c6848f547e4abd namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 74f7ea4a-6de7-4a2a-be10-3ef872acde82 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 44decbbf-7379-459a-882b-7789cce310b1 kind: Event lastTimestamp: "2026-07-28T17:34:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:58Z" name: success-200-isvc-22e0e-predictor.18c68492d2d34d00 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 3109992e-a531-4ac9-ace1-ccd14fd11d2d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 44decbbf-7379-459a-882b-7789cce310b1 kind: Event lastTimestamp: "2026-07-28T17:34:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:58Z" name: success-200-isvc-22e0e-predictor.18c68492d3441960 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: a5959dc6-b581-401a-b4b7-27c22f2234a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:35:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 44decbbf-7379-459a-882b-7789cce310b1 kind: Event lastTimestamp: "2026-07-28T17:35:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:28Z" name: success-200-isvc-22e0e-predictor.18c6849d4fb9efe8 namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: a4ecab5b-e750-4f50-8b8c-2344a8dc1874 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:35:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-22e0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 44decbbf-7379-459a-882b-7789cce310b1 kind: Event lastTimestamp: "2026-07-28T17:35:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:28Z" name: success-200-isvc-22e0e-predictor.18c6849d50232f2f namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: e236c610-fe3a-40db-b760-b504a8da46cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-22e0e namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 71b658d3-637f-4f67-819d-2436234f5e5d kind: Event lastTimestamp: "2026-07-28T17:35:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:00Z" name: success-200-isvc-22e0e.18c6848f5478386c namespace: kserve-ci-e2e-test resourceVersion: "16114" uid: 35fac112-e5a5-4d0c-8a93-38f660fbfd47 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-22e0e namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 71b658d3-637f-4f67-819d-2436234f5e5d kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: 'Failed to update status for InferenceService "success-200-isvc-22e0e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-22e0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e.18c6848f5564f6ef namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: 2c7a136f-bdf5-4669-9210-44d67e0bf687 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-22e0e namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 71b658d3-637f-4f67-819d-2436234f5e5d kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-22e0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-22e0e.18c6848f556585e0 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 7f862338-4d4e-460a-8944-d5b84aadd9dc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-22e0e namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 71b658d3-637f-4f67-819d-2436234f5e5d kind: Event lastTimestamp: "2026-07-28T17:35:00Z" message: InferenceService [success-200-isvc-22e0e] is Ready metadata: creationTimestamp: "2026-07-28T17:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:00Z" name: success-200-isvc-22e0e.18c6849a501baff4 namespace: kserve-ci-e2e-test resourceVersion: "16112" uid: b0a5f76e-fe2c-455e-86e7-3986b7ee3ac2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:44:39.058716Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-614a5-predictor-776c647d45-4xwq8 to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c68520fec487fd namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: b047905d-09fa-4cbf-8f11-043240e2002e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c6852119101d1f namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: 9d5e9b48-cbc3-4730-b968-daa5576f6644 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c685211a859833 namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 2bcb0175-4e65-4c5b-ad33-1986a1aa8428 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c685211e1a8212 namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: bee8cb31-11e8-4e56-8a8f-05db80969e77 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c685211e8f1703 namespace: kserve-ci-e2e-test resourceVersion: "20953" uid: 5eb7c0c3-b9db-4ab0-b205-af3c19658c36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c685211e99bb78 namespace: kserve-ci-e2e-test resourceVersion: "20954" uid: d0230b45-4531-476e-9125-da52ddabe668 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c6852121bc0170 namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: 606b6ff6-e6cc-4893-b00a-3705e1ed7313 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c68521222e60df namespace: kserve-ci-e2e-test resourceVersion: "20956" uid: b6f14a75-cbb0-42ee-8b05-dff8525d1802 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:53:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:55Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c6852122952a50 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: 9e18a720-5d10-44fe-a37e-1acdabbe2efe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:53Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c685a21b82dff4 namespace: kserve-ci-e2e-test resourceVersion: "25271" uid: bc0da69a-a59a-4be5-bfef-ab6b5d8d6bae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:53Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c685a21b83601b namespace: kserve-ci-e2e-test resourceVersion: "25274" uid: 30a59148-c132-4212-bcf7-827a4b2619f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-614a5-predictor-776c647d45-4xwq8 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 0da81b68-4834-4e53-b9b1-1305fcec6bc6 kind: Event lastTimestamp: "2026-07-28T17:53:55Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:55Z" name: success-200-isvc-614a5-predictor-776c647d45-4xwq8.18c685a297332df1 namespace: kserve-ci-e2e-test resourceVersion: "25378" uid: 8271a7d0-af1a-4cba-9d3e-d2052a363c4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-614a5-predictor-776c647d45 namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 78b49725-6efa-4417-b2fd-5e93a992ce15 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: 'Created pod: success-200-isvc-614a5-predictor-776c647d45-4xwq8' metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-614a5-predictor-776c647d45.18c68520fc34dfae namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: 8db5ff51-c703-43ec-95c4-b1e2304a0594 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: fd3cdbdf-63d6-4613-a323-d9e8f78253b9 kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Scaled up replica set success-200-isvc-614a5-predictor-776c647d45 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:38Z" name: success-200-isvc-614a5-predictor.18c68520f6cae255 namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: f16e3d19-ff73-4016-b110-25784d8a81f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: c3892df9-f304-46a1-8bec-0286b2e388da kind: Event lastTimestamp: "2026-07-28T17:45:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:08Z" name: success-200-isvc-614a5-predictor.18c6852473c67d75 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: daab5d33-e79b-44b1-87e4-27942d02db91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: c3892df9-f304-46a1-8bec-0286b2e388da kind: Event lastTimestamp: "2026-07-28T17:45:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:08Z" name: success-200-isvc-614a5-predictor.18c68524742efb5e namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: 72173f7d-897c-49d2-b1b9-19cdb0d8e444 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:45:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: c3892df9-f304-46a1-8bec-0286b2e388da kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:53Z" name: success-200-isvc-614a5-predictor.18c6852b71a123eb namespace: kserve-ci-e2e-test resourceVersion: "21855" uid: 36938c60-61ae-45ef-b169-b8cafbbe4d95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:45:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-614a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: c3892df9-f304-46a1-8bec-0286b2e388da kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:53Z" name: success-200-isvc-614a5-predictor.18c6852b7212853c namespace: kserve-ci-e2e-test resourceVersion: "21858" uid: 621fc7ff-478b-4d81-af91-09e3b93bfcd0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-614a5 namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: 007b1e58-46c7-4159-83ca-52969384eeee kind: Event lastTimestamp: "2026-07-28T17:45:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:25Z" name: success-200-isvc-614a5.18c68520f5c17b26 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: c60fe980-e31f-4544-a112-79a290add4a1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-614a5 namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 007b1e58-46c7-4159-83ca-52969384eeee kind: Event lastTimestamp: "2026-07-28T17:45:25Z" message: InferenceService [success-200-isvc-614a5] is Ready metadata: creationTimestamp: "2026-07-28T17:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:25Z" name: success-200-isvc-614a5.18c6852bdfdcaac7 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 482dd249-1c0c-42ee-b6a2-fbda4bd8682b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:44:38.285544Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-61a23-predictor-644f77f65c-7gb8k to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:44:38Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c68520d0aee110 namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: f48bdec1-12e1-439a-a715-6e84e75c61dc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-61a23-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:38Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c68520da2656f9 namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 8e0ccaef-60ac-4f0e-a068-b01c417bfc64 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:44:40Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c68521435c2198 namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: 480a17f8-09d3-426c-801c-0a46f2c571da reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:40Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c6852144fc61e6 namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 885fa797-673f-4a5c-b22c-63fbf63b20b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:40Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c68521482b9cb3 namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 86b06d6d-9b0e-4eea-868b-53b05bccbd56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:40Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c685214899fba3 namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: cd6fa66d-6708-461d-85ba-6075b485ecb4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:40Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c6852148a4a7d6 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: e4e013c3-fe34-40f5-89e4-4667b2109f9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:40Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c685214c433429 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 356fd59c-ee67-492a-9bb1-daf93eef872d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:40Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c685214cb3fb54 namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 90157503-685e-4f09-b1ec-ca01fc8a94ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:45:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:17Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c685219a794c8a namespace: kserve-ci-e2e-test resourceVersion: "21327" uid: 6a436394-e489-4be7-babb-859f4fea5295 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:52Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c68532163aeb25 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: e66bcda7-a8d3-44ef-91c4-7af1e4b02dde reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:52Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c68532163b82c0 namespace: kserve-ci-e2e-test resourceVersion: "21751" uid: 6ff05e8e-9018-455d-83fb-a77e8122b583 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52699f3f-9cb0-45a0-a3ba-f1a1a1c5a1ca kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:52Z" name: success-200-isvc-61a23-predictor-644f77f65c-7gb8k.18c6853222dc44a6 namespace: kserve-ci-e2e-test resourceVersion: "21810" uid: abff75f5-52e4-4a5a-9093-f754c7fc7546 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-61a23-predictor-644f77f65c namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: fcb9606f-9c3b-48e1-a150-2035048d70cb kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: 'Created pod: success-200-isvc-61a23-predictor-644f77f65c-7gb8k' metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:38Z" name: success-200-isvc-61a23-predictor-644f77f65c.18c68520d000c1bf namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 5e295979-04e3-4705-9848-7112201a6e8f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: 3aeca758-4e39-48ea-8055-6c3632d50f10 kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Scaled up replica set success-200-isvc-61a23-predictor-644f77f65c from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:38Z" name: success-200-isvc-61a23-predictor.18c68520ccff5069 namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: dd4e21ab-d4ae-4bc7-b3e4-fd054b22063c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20736" uid: 7a4af5cf-0192-4f8a-b46b-98121fe47591 kind: Event lastTimestamp: "2026-07-28T17:44:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:53Z" name: success-200-isvc-61a23-predictor.18c685244ba6ab81 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: d55420a6-b323-48ff-9011-23be0c0fe633 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "20736" uid: 7a4af5cf-0192-4f8a-b46b-98121fe47591 kind: Event lastTimestamp: "2026-07-28T17:44:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:53Z" name: success-200-isvc-61a23-predictor.18c685244c11f734 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: b50d4909-35ef-4ce0-8676-0010638a53f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:45:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: 7a4af5cf-0192-4f8a-b46b-98121fe47591 kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:38Z" name: success-200-isvc-61a23-predictor.18c68527caedbcd6 namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: 271d6701-28be-476f-a98d-be7a07ea4fba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:45:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-61a23-predictor namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: 7a4af5cf-0192-4f8a-b46b-98121fe47591 kind: Event lastTimestamp: "2026-07-28T17:45:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:38Z" name: success-200-isvc-61a23-predictor.18c68527cb699018 namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: 44251fcc-3e82-4ea8-bf73-0349112d0341 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-61a23 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: a5634274-10b4-4f7f-9656-57a4c5d6741b kind: Event lastTimestamp: "2026-07-28T17:45:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:27Z" name: success-200-isvc-61a23.18c68520ccb3923f namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: e3fd0eb3-141a-4be7-b45e-360d4292aeab reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-61a23 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: a5634274-10b4-4f7f-9656-57a4c5d6741b kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: 'Failed to update status for InferenceService "success-200-isvc-61a23": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-61a23": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:38Z" name: success-200-isvc-61a23.18c68520cd593e96 namespace: kserve-ci-e2e-test resourceVersion: "20741" uid: a0aa9e3e-dff4-4eff-b709-f9a2b76d4a58 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-61a23 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: a5634274-10b4-4f7f-9656-57a4c5d6741b kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-61a23": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:38Z" name: success-200-isvc-61a23.18c68520cd598867 namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: e7d45ca4-ac96-4f23-a517-1038f119fef2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-61a23 namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: a5634274-10b4-4f7f-9656-57a4c5d6741b kind: Event lastTimestamp: "2026-07-28T17:45:27Z" message: InferenceService [success-200-isvc-61a23] is Ready metadata: creationTimestamp: "2026-07-28T17:45:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:27Z" name: success-200-isvc-61a23.18c6852c57291988 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 891a15c3-9a45-4ca1-b4d5-c9ff118f5f99 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:45:52.509728Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21755" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-787a1-predictor-869dbdb759-qnbjk to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:45:52Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c6853218ca8c70 namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: 93e74d70-bd83-419d-873d-d6d968896c2e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-787a1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:52Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c685321f55fd37 namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: b7470394-ae6b-4b07-a5fd-318c1d4a0063 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:45:53Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c68532443d9ab9 namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: 5e1e1949-9cba-40ca-91a3-a66edd8ca8c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c68532459c6ab8 namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 3ecee853-dfc9-4190-8521-c06eb41f0a23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c6853248a20467 namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 2732b9e3-e55f-4778-bf7c-032eecf1d224 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c685324911774b namespace: kserve-ci-e2e-test resourceVersion: "21841" uid: 77964ccf-0b51-46c7-93eb-44b8d1b474fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c68532491a82f3 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 5e66e496-cff0-4f9f-84a2-d11a33d031b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c685324b9dc367 namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: 08f48980-a66e-46ab-b9c4-c61724713806 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:45:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:45:53Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c685324c57131e namespace: kserve-ci-e2e-test resourceVersion: "21844" uid: ec32120b-29fd-4366-a7b8-cf4c7cdbaec4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:55:09Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:09Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c685326ebbb96e namespace: kserve-ci-e2e-test resourceVersion: "26573" uid: 2ad95da5-adab-47ea-8926-246932f37966 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:07Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c685b34e319779 namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: f18db2f3-dd40-435d-a555-b7c4699f697a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:07Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c685b34e31fe57 namespace: kserve-ci-e2e-test resourceVersion: "26482" uid: 10fdbeff-9c7d-460e-9b7f-751790a5504a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 39137e71-bcda-4ba3-bdca-60c6e8a91b1a kind: Event lastTimestamp: "2026-07-28T17:55:09Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:09Z" name: success-200-isvc-787a1-predictor-869dbdb759-qnbjk.18c685b3e31a7dbc namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: 14e71e61-6d68-4a7e-9fce-50bf018f9e88 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-787a1-predictor-869dbdb759 namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: 007aedc0-9b95-4b7d-9ccd-910d0bf7d401 kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: 'Created pod: success-200-isvc-787a1-predictor-869dbdb759-qnbjk' metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:52Z" name: success-200-isvc-787a1-predictor-869dbdb759.18c68532180b13e1 namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: 30999b8a-f989-40c9-97b4-ea4e6ba244b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 0d3c430e-efc7-426c-bcf0-da8e6999b7db kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: Scaled up replica set success-200-isvc-787a1-predictor-869dbdb759 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:52Z" name: success-200-isvc-787a1-predictor.18c6853215a02f9a namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: c3d4ddd5-4635-4b3a-ab60-68b2b965b064 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:46:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: e440be7a-6786-41ab-8031-a4ec57377072 kind: Event lastTimestamp: "2026-07-28T17:46:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:46:37Z" name: success-200-isvc-787a1-predictor.18c68535940237b8 namespace: kserve-ci-e2e-test resourceVersion: "22197" uid: e74acb4b-add8-474f-9090-0742848358bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:46:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: e440be7a-6786-41ab-8031-a4ec57377072 kind: Event lastTimestamp: "2026-07-28T17:46:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:46:37Z" name: success-200-isvc-787a1-predictor.18c68535947cd69c namespace: kserve-ci-e2e-test resourceVersion: "22198" uid: 6dcf5488-3fda-47b1-a5a2-e15c480c0392 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:46:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: e440be7a-6786-41ab-8031-a4ec57377072 kind: Event lastTimestamp: "2026-07-28T17:47:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:07Z" name: success-200-isvc-787a1-predictor.18c6854011ded5b1 namespace: kserve-ci-e2e-test resourceVersion: "22467" uid: bc00b451-ced2-4df2-9c30-4e88c734824b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:46:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-787a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: e440be7a-6786-41ab-8031-a4ec57377072 kind: Event lastTimestamp: "2026-07-28T17:47:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:07Z" name: success-200-isvc-787a1-predictor.18c6854012813af3 namespace: kserve-ci-e2e-test resourceVersion: "22470" uid: 482fc61b-b50c-41c6-88bc-7258135c8069 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-787a1 namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 69ee4e73-2c0a-4c2e-b9b4-e1c39aa7204e kind: Event lastTimestamp: "2026-07-28T17:46:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:46:40Z" name: success-200-isvc-787a1.18c6853215be8ceb namespace: kserve-ci-e2e-test resourceVersion: "22239" uid: 96b0dd35-427b-4105-a45a-a1f18088ba21 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-787a1 namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 69ee4e73-2c0a-4c2e-b9b4-e1c39aa7204e kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-787a1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-787a1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:52Z" name: success-200-isvc-787a1.18c68532165356cf namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: 37af2106-3877-45d6-a039-8e1475fd5c69 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-787a1 namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 69ee4e73-2c0a-4c2e-b9b4-e1c39aa7204e kind: Event lastTimestamp: "2026-07-28T17:45:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-787a1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:45:52Z" name: success-200-isvc-787a1.18c685321653c2a2 namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: db1b9372-f71a-4e96-8f92-609b164069e9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:46:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-787a1 namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 69ee4e73-2c0a-4c2e-b9b4-e1c39aa7204e kind: Event lastTimestamp: "2026-07-28T17:46:40Z" message: InferenceService [success-200-isvc-787a1] is Ready metadata: creationTimestamp: "2026-07-28T17:46:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:46:40Z" name: success-200-isvc-787a1.18c6853d2a52af88 namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: b76c3a19-2f1d-4c50-8dba-aad99eeeb74a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:55:04.039109Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7d999-predictor-7f48846ddc-d5stl to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:55:04Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b282809c81 namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: c5dc30b1-1802-48a8-8dfc-83f68e984ac4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T17:55:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7d999-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:04Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b28f41baa2 namespace: kserve-ci-e2e-test resourceVersion: "26350" uid: 867aba41-169c-4299-b8a2-68fe786c9f89 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:55:05Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b2bffdba9a namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 004655f2-e089-4d8b-8559-63d8529462d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b2c20c5f50 namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: 6e94f98e-2421-4196-8cfa-ee0b899be40f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b2c5096e7b namespace: kserve-ci-e2e-test resourceVersion: "26369" uid: 4ffc78a1-0c11-4755-b776-e6d9d8517793 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b2c5841760 namespace: kserve-ci-e2e-test resourceVersion: "26370" uid: 99644e73-a953-40f5-86de-8e589c28e14b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b2c58fdab8 namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: dae7b853-308a-4a60-a516-59b8cdd254d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b2c8ccedd4 namespace: kserve-ci-e2e-test resourceVersion: "26372" uid: 4395d743-7a1b-4f28-979e-570528f074e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T17:55:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:05Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b2c950772e namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: b91b1008-fa62-4e6a-911b-48cfd53bcdf2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T17:55:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:41Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c685b2f62eeb08 namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: ce6219cc-11a2-44df-9d30-996d06c6bde6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:04:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T18:04:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:04:18Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c68633b0f1b442 namespace: kserve-ci-e2e-test resourceVersion: "30605" uid: d775311d-245a-46ed-b3f8-8a3d02357e88 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:04:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8155544f-2dae-4598-962f-97a2c83dcdf2 kind: Event lastTimestamp: "2026-07-28T18:04:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:04:18Z" name: success-200-isvc-7d999-predictor-7f48846ddc-d5stl.18c68633b0f227be namespace: kserve-ci-e2e-test resourceVersion: "30606" uid: 19f4cea5-84b5-4994-9e0b-28414292c836 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7d999-predictor-7f48846ddc namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: 5a8baf67-1f40-4252-9e4f-4e5f26d191a0 kind: Event lastTimestamp: "2026-07-28T17:55:04Z" message: 'Created pod: success-200-isvc-7d999-predictor-7f48846ddc-d5stl' metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:04Z" name: success-200-isvc-7d999-predictor-7f48846ddc.18c685b2823f5fec namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 244bd588-a38d-4b4a-9f6e-ebf6725be0b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: a63f9f4f-66d9-4600-a044-14203abf5895 kind: Event lastTimestamp: "2026-07-28T17:55:03Z" message: Scaled up replica set success-200-isvc-7d999-predictor-7f48846ddc from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:03Z" name: success-200-isvc-7d999-predictor.18c685b27ee92138 namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: cb4bb94a-0687-4e08-b808-965832526388 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26281" uid: fab4a043-93f3-488c-ba77-8ecc1a22475d kind: Event lastTimestamp: "2026-07-28T17:55:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:34Z" name: success-200-isvc-7d999-predictor.18c685b5fdf249c5 namespace: kserve-ci-e2e-test resourceVersion: "26788" uid: db27b964-60e9-4a82-9554-1e6fbc69e085 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26281" uid: fab4a043-93f3-488c-ba77-8ecc1a22475d kind: Event lastTimestamp: "2026-07-28T17:55:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:34Z" name: success-200-isvc-7d999-predictor.18c685b5fe60dbed namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 71dcbd7e-4cb0-4035-8dec-6d9a1c54ed98 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:55:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26658" uid: fab4a043-93f3-488c-ba77-8ecc1a22475d kind: Event lastTimestamp: "2026-07-28T17:56:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:55:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:19Z" name: success-200-isvc-7d999-predictor.18c685bcfc31b95f namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: 171e5f39-d787-4fab-820a-3a072f3ec11c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:55:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d999-predictor namespace: kserve-ci-e2e-test resourceVersion: "26658" uid: fab4a043-93f3-488c-ba77-8ecc1a22475d kind: Event lastTimestamp: "2026-07-28T17:56:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:55:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:19Z" name: success-200-isvc-7d999-predictor.18c685bcfca79c59 namespace: kserve-ci-e2e-test resourceVersion: "27259" uid: 789ffa69-c3e5-499c-8c4e-f18699217872 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:55:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d999 namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: 2c2db432-ffe7-4d99-9d48-ce02d6aa8751 kind: Event lastTimestamp: "2026-07-28T17:55:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:52Z" name: success-200-isvc-7d999.18c685b27f7ec23d namespace: kserve-ci-e2e-test resourceVersion: "26949" uid: fcbf71e7-9915-49c4-9291-9121842e6ee9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d999 namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: 2c2db432-ffe7-4d99-9d48-ce02d6aa8751 kind: Event lastTimestamp: "2026-07-28T17:55:03Z" message: 'Failed to update status for InferenceService "success-200-isvc-7d999": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7d999": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:04Z" name: success-200-isvc-7d999.18c685b2800e3cba namespace: kserve-ci-e2e-test resourceVersion: "26291" uid: 9f0a4ef6-7234-4c79-9e96-6ca20e9c231c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d999 namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: 2c2db432-ffe7-4d99-9d48-ce02d6aa8751 kind: Event lastTimestamp: "2026-07-28T17:55:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7d999": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:04Z" name: success-200-isvc-7d999.18c685b2800e9ca2 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 8d5ea17a-9fa4-4ce6-8707-7e001a9612f0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d999 namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 2c2db432-ffe7-4d99-9d48-ce02d6aa8751 kind: Event lastTimestamp: "2026-07-28T17:55:52Z" message: InferenceService [success-200-isvc-7d999] is Ready metadata: creationTimestamp: "2026-07-28T17:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:52Z" name: success-200-isvc-7d999.18c685bdb545ff35 namespace: kserve-ci-e2e-test resourceVersion: "26948" uid: 3c5a2621-f39a-4be4-bca6-55b7e2006230 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:28.318494Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:28Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c6851086527c96 namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: 6aa7e79e-fed4-466b-bb5c-a0ecd02ceda5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8b8b5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:28Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c6851091c7ef13 namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 6ad992da-16e1-4ebe-9553-857b023c55a0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:29Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c68510c3857cd2 namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: 0583d7ee-cecb-4421-a4f2-1a921c9e2c59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c68510c533ad2c namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: 3662950c-0821-499d-bbca-e4c04ada7265 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c68510c80adb50 namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: 5f55d088-4b82-4d21-9f1c-2dabf29a18f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c68510c87c5368 namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: 47ab23a8-f47d-4f37-b53c-4a9f63cf9b15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c68510c8857b8b namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 7d6f31b1-ab50-4d13-a499-a5b0c579bd38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c68510cb3d86c4 namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: 73af7003-7ebd-4a99-94a7-96244bc490b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:43:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:29Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c68510cbaa0529 namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: 721ad5ed-bf4b-4554-ac60-122c76c79a52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:44:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:07Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c685113ee0e64c namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: e726a971-a9e8-4330-94b9-51687fad3886 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c6852107827b8a namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 17d4c08c-ce06-4186-aaf0-24db2561d40b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:44:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:39Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c6852107832801 namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: 772bec31-b3a8-4085-b0eb-983a707f40e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: c561db9f-a9b5-4c4f-87e5-4d5d5df07782 kind: Event lastTimestamp: "2026-07-28T17:44:42Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:42Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk.18c68521c71d930e namespace: kserve-ci-e2e-test resourceVersion: "21016" uid: 5c5b55a6-8f8d-46ab-98ec-9a6398a39657 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8b8b5-predictor-7f4bbc59f7 namespace: kserve-ci-e2e-test resourceVersion: "19999" uid: 24c91de9-d63f-4678-ba15-89bdd3bb1cd9 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: 'Created pod: success-200-isvc-8b8b5-predictor-7f4bbc59f7-2hshk' metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:28Z" name: success-200-isvc-8b8b5-predictor-7f4bbc59f7.18c6851085c2ab7c namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: 3926076d-2370-49e1-988e-bece9cd07e93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19997" uid: c5fe254a-2b74-4f01-92aa-061cd21909f8 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: Scaled up replica set success-200-isvc-8b8b5-predictor-7f4bbc59f7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:28Z" name: success-200-isvc-8b8b5-predictor.18c6851080a21605 namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: d183b642-531d-416b-8b55-312cfa351a07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: 2a8136d1-faeb-4f72-95c5-7598ec11509d kind: Event lastTimestamp: "2026-07-28T17:44:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:13Z" name: success-200-isvc-8b8b5-predictor.18c68513ff2e3fe2 namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 84007ca3-e69a-412d-9ae3-812dc0d51fa4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: 2a8136d1-faeb-4f72-95c5-7598ec11509d kind: Event lastTimestamp: "2026-07-28T17:44:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:13Z" name: success-200-isvc-8b8b5-predictor.18c68513ffa71420 namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: e335d489-e1aa-4c90-b041-5de85fa74d5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 2a8136d1-faeb-4f72-95c5-7598ec11509d kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:28Z" name: success-200-isvc-8b8b5-predictor.18c685177ef552b6 namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: a25584c7-cd8e-45a1-af5a-2b8b7ba4e788 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b8b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 2a8136d1-faeb-4f72-95c5-7598ec11509d kind: Event lastTimestamp: "2026-07-28T17:44:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:28Z" name: success-200-isvc-8b8b5-predictor.18c685177fbe7c05 namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: 06480239-44c6-4dfd-adbd-f76e2ea23930 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: ada55208-8751-4d91-af35-b4efc5b42095 kind: Event lastTimestamp: "2026-07-28T17:44:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:17Z" name: success-200-isvc-8b8b5.18c6851080a0e4b2 namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: f6a9a072-4cef-4920-a9a4-73dbad094ba2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: ada55208-8751-4d91-af35-b4efc5b42095 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-8b8b5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8b8b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:28Z" name: success-200-isvc-8b8b5.18c68510815aee58 namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: 9267c709-5577-4ccb-8cc8-c145e8dee213 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: ada55208-8751-4d91-af35-b4efc5b42095 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8b8b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:28Z" name: success-200-isvc-8b8b5.18c68510815b459f namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: 4a75213e-3660-42c4-8dc0-acba433de1ec reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b8b5 namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: ada55208-8751-4d91-af35-b4efc5b42095 kind: Event lastTimestamp: "2026-07-28T17:44:17Z" message: InferenceService [success-200-isvc-8b8b5] is Ready metadata: creationTimestamp: "2026-07-28T17:44:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:17Z" name: success-200-isvc-8b8b5.18c6851bfb429482 namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: afd18202-cb11-4789-a709-cec49998390a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:35:23.329657Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c6849f9abc0984 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 42764959-54b4-49ee-8e62-928df4a3b65b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c6849fb3f1a57c namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 4ba5ad2a-d9d2-4796-9bea-b3531d343d05 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c6849fb54ad925 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 8bc681c1-f908-4417-aa7a-c662be14a525 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c6849fb7e58b0a namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 124a43ec-8ba3-469c-b627-d3c662b2d97c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c6849fb8579bfa namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 381d75a8-b0e9-44ce-b072-d716c7f58af0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c6849fb861f8c5 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: bbabad53-3c0f-4af8-81b2-eac43408c5ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c6849fbaf143d0 namespace: kserve-ci-e2e-test resourceVersion: "16512" uid: 5f73462b-d33d-4ffb-959e-00e4af6bab05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c6849fbb731bc5 namespace: kserve-ci-e2e-test resourceVersion: "16513" uid: 4a2fcae2-0eb8-471c-99b9-6f2cab73e18a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:35:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:35:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:40Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c6849feef02023 namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 3b606877-b8cb-4604-acb7-45764bb79b0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:38Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c68520d6b30926 namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: a2a640e5-31c2-499c-9817-2f8793beb2a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:44:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:38Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c68520d6b3721d namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: 76dc9951-c278-48a2-afaf-ff078e46245a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: d4b3a598-cb9d-46a5-a17c-481a7e45fddb kind: Event lastTimestamp: "2026-07-28T17:44:40Z" message: 'Readiness probe failed: Get "https://10.133.0.19:8643/healthz": dial tcp 10.133.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:40Z" name: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj.18c685216350e93f namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: c859de36-23b6-420a-92a5-24ee87d8a32c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b1dfe-predictor-559469fcc9 namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 636d4a6c-0141-43ed-b947-d04b10b42b8d kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: 'Created pod: success-200-isvc-b1dfe-predictor-559469fcc9-cjnnj' metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor-559469fcc9.18c6849f9a3de07c namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: 71adacd9-a894-4a1c-a86c-6f2ab0b03c9a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 370fb520-3ebf-48d4-953b-2d4bc4849ea5 kind: Event lastTimestamp: "2026-07-28T17:35:23Z" message: Scaled up replica set success-200-isvc-b1dfe-predictor-559469fcc9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:23Z" name: success-200-isvc-b1dfe-predictor.18c6849f9841bc1a namespace: kserve-ci-e2e-test resourceVersion: "16430" uid: 3b8cedec-8c8b-402e-8dc7-9cc4592ed868 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 33bf52a3-49a7-4ac2-9564-2ca174eff7cb kind: Event lastTimestamp: "2026-07-28T17:35:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:38Z" name: success-200-isvc-b1dfe-predictor.18c684a317105f4b namespace: kserve-ci-e2e-test resourceVersion: "16657" uid: 870f5362-1129-4e67-a763-041053189e74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 33bf52a3-49a7-4ac2-9564-2ca174eff7cb kind: Event lastTimestamp: "2026-07-28T17:35:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:38Z" name: success-200-isvc-b1dfe-predictor.18c684a317a08c72 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: ad12de76-047b-43ef-aa8a-b8e9195b0a5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:35:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: 33bf52a3-49a7-4ac2-9564-2ca174eff7cb kind: Event lastTimestamp: "2026-07-28T17:36:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:38Z" name: success-200-isvc-b1dfe-predictor.18c684a696c8af50 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 93b3fb07-5fc7-4d40-abb8-67a65066cef2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:35:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b1dfe-predictor namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: 33bf52a3-49a7-4ac2-9564-2ca174eff7cb kind: Event lastTimestamp: "2026-07-28T17:36:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:36:38Z" name: success-200-isvc-b1dfe-predictor.18c684a6971fceb9 namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: 17e505b6-8408-4145-b145-15d9096e352a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:35:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: b3db4503-4f1f-4acd-9ccd-b72f4767cdb5 kind: Event lastTimestamp: "2026-07-28T17:36:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:36:10Z" name: success-200-isvc-b1dfe.18c6849f985253bf namespace: kserve-ci-e2e-test resourceVersion: "16921" uid: 50878120-b35f-4b47-9bd3-aaecbace918a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:36:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b1dfe namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: b3db4503-4f1f-4acd-9ccd-b72f4767cdb5 kind: Event lastTimestamp: "2026-07-28T17:36:10Z" message: InferenceService [success-200-isvc-b1dfe] is Ready metadata: creationTimestamp: "2026-07-28T17:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:36:10Z" name: success-200-isvc-b1dfe.18c684aaaaef76be namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: 12e79355-ae5e-49b6-b34a-5639c4f5338a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:55:07.362255Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cb1ee-predictor-7b757b875f-46rjc to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:55:07Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b34893b452 namespace: kserve-ci-e2e-test resourceVersion: "26449" uid: 71149d49-5d4e-4e6f-ab35-471e65f7387e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-cb1ee-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:07Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b355aeb026 namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: 2a560f83-ec90-4684-a190-c9f3bc3c7512 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:55:08Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b3863d5921 namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 5b63898b-4c5a-4605-9d5e-8d2886720e7a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b388465617 namespace: kserve-ci-e2e-test resourceVersion: "26544" uid: c6cb5e13-23af-417a-8105-7ff68a70a640 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b38c662fb6 namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 7f7b3046-2a6a-4653-abba-a6e577d58fc8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b38ce8d0ed namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: f10e80bc-044a-49e6-bdef-3d54a72d50c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b38cf5aa31 namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 58d37319-c8a5-4fcf-8779-426e52bc2dd6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b38ffc0fdd namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 3325150d-7d34-4c6f-a6a5-67ae31513d0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T17:55:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:08Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b390759100 namespace: kserve-ci-e2e-test resourceVersion: "26552" uid: d492f1a0-b431-415a-afe9-aaf4a6e03ab6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:55:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T17:55:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:55Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c685b3aa48e853 namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: bee5a8f5-d8fa-435a-9c0f-ff69dcb85ed1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T18:12:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:12:36Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c686a79eb28680 namespace: kserve-ci-e2e-test resourceVersion: "34055" uid: 181382ee-a9d1-4aeb-a3a7-95730d290c28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: b1027091-27cb-46ce-ae3d-ecdcfe582ca3 kind: Event lastTimestamp: "2026-07-28T18:12:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:12:36Z" name: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc.18c686a79eb3483a namespace: kserve-ci-e2e-test resourceVersion: "34056" uid: 95585576-78fb-481a-a559-9bb4c8b8ba4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cb1ee-predictor-7b757b875f namespace: kserve-ci-e2e-test resourceVersion: "26436" uid: 396abaa1-079c-4163-995a-9cfaa77d180b kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: 'Created pod: success-200-isvc-cb1ee-predictor-7b757b875f-46rjc' metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:07Z" name: success-200-isvc-cb1ee-predictor-7b757b875f.18c685b348049184 namespace: kserve-ci-e2e-test resourceVersion: "26446" uid: a70a9133-fb7d-4c4b-bc5c-e73d81777a91 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 650a02d3-4f12-4ef9-a86e-aa32e496388e kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: Scaled up replica set success-200-isvc-cb1ee-predictor-7b757b875f from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:07Z" name: success-200-isvc-cb1ee-predictor.18c685b3455154e9 namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 3b2a6053-937e-403b-9aa2-308e8079b2a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:55:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26435" uid: 4264bc3d-3288-4680-a26e-a53f8a4d6b9f kind: Event lastTimestamp: "2026-07-28T17:55:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:52Z" name: success-200-isvc-cb1ee-predictor.18c685b6c49dacf7 namespace: kserve-ci-e2e-test resourceVersion: "26951" uid: dcedf364-fef6-43d0-89e4-884d90b42a00 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:55:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26435" uid: 4264bc3d-3288-4680-a26e-a53f8a4d6b9f kind: Event lastTimestamp: "2026-07-28T17:55:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:55:52Z" name: success-200-isvc-cb1ee-predictor.18c685b6c4f65f04 namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: a0f9c91f-ebf5-4698-a21a-f1e5f2fd0398 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:56:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 4264bc3d-3288-4680-a26e-a53f8a4d6b9f kind: Event lastTimestamp: "2026-07-28T17:56:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:56:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:22Z" name: success-200-isvc-cb1ee-predictor.18c685c142ab7a7c namespace: kserve-ci-e2e-test resourceVersion: "27289" uid: 39f1739e-01af-4e11-8e14-d0de7c9d91ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:56:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 4264bc3d-3288-4680-a26e-a53f8a4d6b9f kind: Event lastTimestamp: "2026-07-28T17:56:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:56:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:22Z" name: success-200-isvc-cb1ee-predictor.18c685c1433ec1f6 namespace: kserve-ci-e2e-test resourceVersion: "27290" uid: ae823ff6-55f6-4c3c-9e35-f17d28960a2c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: d9fd4552-ce8d-444c-ad27-a6a3cabfc85f kind: Event lastTimestamp: "2026-07-28T17:56:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:56:05Z" name: success-200-isvc-cb1ee.18c685b3458e55ec namespace: kserve-ci-e2e-test resourceVersion: "27108" uid: 610cb079-bf4a-4b70-a3ea-146fff3eabe5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: d9fd4552-ce8d-444c-ad27-a6a3cabfc85f kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: 'Failed to update status for InferenceService "success-200-isvc-cb1ee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cb1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:07Z" name: success-200-isvc-cb1ee.18c685b34643d5e9 namespace: kserve-ci-e2e-test resourceVersion: "26441" uid: fc65710e-0205-443f-83a0-4d6b1a43815e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: d9fd4552-ce8d-444c-ad27-a6a3cabfc85f kind: Event lastTimestamp: "2026-07-28T17:55:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cb1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:55:07Z" name: success-200-isvc-cb1ee.18c685b3464414de namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: c349b33a-8d7e-4132-8830-b811c7067673 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "27106" uid: d9fd4552-ce8d-444c-ad27-a6a3cabfc85f kind: Event lastTimestamp: "2026-07-28T17:56:05Z" message: InferenceService [success-200-isvc-cb1ee] is Ready metadata: creationTimestamp: "2026-07-28T17:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:56:05Z" name: success-200-isvc-cb1ee.18c685c0ba77ae08 namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: 0eff37a8-f025-410e-b944-12ca576d047c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:39.013316Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f0903-predictor-b88cb88c-8g2fn to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:39Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847958b7b6ba namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 6a0b0b13-9f10-4d22-874f-a1053f9bee32 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f0903-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:39Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c684796639fa4b namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: 24ecf472-8d1b-4938-bba8-99ed5ff75ca7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Add eth0 [10.133.0.14/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:40Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847995923289 namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 3f65f873-cade-4cfe-b4a3-1ef7b9fb1a32 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:40Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:40Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c68479970c447a namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: f24fb139-32ac-490d-9fd1-f0244890cd14 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" in 11.596s (11.596s including waiting). Image size: 1334926941 bytes.' metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:51Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847c4a40c8b3 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 93e6a31d-be5c-4b88-9159-4d20401d3a79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:51Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847c4dbd4684 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: a05c8684-ba4d-4672-b7ee-1dc9bd3e92c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:51Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847c4e2c416f namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 128791d3-2958-4fac-8114-deeb3bd3eb0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:51Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:51Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847c4e38e014 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: a0d54654-5e7e-4707-8080-cc209e5a9d02 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:53Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.233s (2.233s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:53Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847cd3613294 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: d2f8e006-41ca-4413-ab16-d14c8848c2c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:54Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847cd6837930 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: f41ca951-e621-4504-aa9c-ef992115115e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:32:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:54Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847cd6f69f05 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: b09b9c81-3221-4943-beff-1416798adf8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:33:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.14:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:30Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6847ce3d45fbc namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: 6695c572-fc7b-4d06-af6d-3cde8c4d95ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6848f55fd134f namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: f50a454c-5bb0-4e07-9da1-a494a4884816 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6848f55fd76af namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: ed0795f0-29cb-4df8-93b7-80d14a8010a7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 00e17ba8-b199-4f61-83e7-9483f2ba0598 kind: Event lastTimestamp: "2026-07-28T17:34:15Z" message: 'Readiness probe failed: Get "https://10.133.0.14:8643/healthz": dial tcp 10.133.0.14:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:15Z" name: success-200-isvc-f0903-predictor-b88cb88c-8g2fn.18c6848fbff8fed1 namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: cfa32f8c-b7ca-4b4a-a9e6-dec72b5886b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f0903-predictor-b88cb88c namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: 081108e5-3e8f-4ceb-8351-aeb298d40c26 kind: Event lastTimestamp: "2026-07-28T17:32:39Z" message: 'Created pod: success-200-isvc-f0903-predictor-b88cb88c-8g2fn' metadata: creationTimestamp: "2026-07-28T17:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:39Z" name: success-200-isvc-f0903-predictor-b88cb88c.18c684795824996b namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: b9acb659-1225-424b-849f-1a383bc5798b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: 6e31d2a2-5b51-4d94-b8b2-5346b3164c7c kind: Event lastTimestamp: "2026-07-28T17:32:38Z" message: Scaled up replica set success-200-isvc-f0903-predictor-b88cb88c from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:38Z" name: success-200-isvc-f0903-predictor.18c6847950146639 namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: f88e0c30-5390-4f9b-9d59-0fc1e4421b8b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:32:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 7d13b87f-3a91-4583-bdb2-711756cd945d kind: Event lastTimestamp: "2026-07-28T17:33:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:39Z" name: success-200-isvc-f0903-predictor.18c6847cd5c33d6a namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: 5fe05527-e8eb-435e-b2b0-e0972927dcd1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:32:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 7d13b87f-3a91-4583-bdb2-711756cd945d kind: Event lastTimestamp: "2026-07-28T17:33:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:39Z" name: success-200-isvc-f0903-predictor.18c6847cd6374c1d namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: 038592d0-59a2-4b21-a3ca-2670788c6abd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:33:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 7d13b87f-3a91-4583-bdb2-711756cd945d kind: Event lastTimestamp: "2026-07-28T17:34:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:09Z" name: success-200-isvc-f0903-predictor.18c6848ad29cb021 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 4e0fb8e0-90eb-4042-8293-b71035615e91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:33:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0903-predictor namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 7d13b87f-3a91-4583-bdb2-711756cd945d kind: Event lastTimestamp: "2026-07-28T17:34:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:33:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:09Z" name: success-200-isvc-f0903-predictor.18c6848ad300d598 namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: b0d65d4b-8201-4760-bac0-61a90d8f289e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0903 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 183797a4-2642-49b4-90ba-1ca75af69de0 kind: Event lastTimestamp: "2026-07-28T17:33:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:40Z" name: success-200-isvc-f0903.18c68479562d502c namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: f0ce1215-caef-4c58-a84e-fa7c3d541f24 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0903 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 183797a4-2642-49b4-90ba-1ca75af69de0 kind: Event lastTimestamp: "2026-07-28T17:32:38Z" message: 'Failed to update status for InferenceService "success-200-isvc-f0903": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f0903": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:38Z" name: success-200-isvc-f0903.18c6847956bf8a42 namespace: kserve-ci-e2e-test resourceVersion: "14491" uid: 6d5b581e-bc45-41d0-80e8-1970c6170442 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0903 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 183797a4-2642-49b4-90ba-1ca75af69de0 kind: Event lastTimestamp: "2026-07-28T17:32:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f0903": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:38Z" name: success-200-isvc-f0903.18c6847956bfded4 namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 6e9de050-f6d0-4fa0-958d-fc14d98f220c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0903 namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 183797a4-2642-49b4-90ba-1ca75af69de0 kind: Event lastTimestamp: "2026-07-28T17:33:40Z" message: InferenceService [success-200-isvc-f0903] is Ready metadata: creationTimestamp: "2026-07-28T17:33:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:40Z" name: success-200-isvc-f0903.18c684879f262fca namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: f1f54cd4-cac2-4466-9e0a-254950978dd4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:53:53.660736Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f7704-predictor-865dcf7dff-bjhft to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:53:53Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685a21f9fc858 namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: 314beca9-f14e-4e2f-988b-4a84baea2b89 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25298" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:53:54Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685a2399955dc namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 189bf798-9a87-494d-b070-9ea890807349 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685a23afbdbac namespace: kserve-ci-e2e-test resourceVersion: "25344" uid: be684ce9-0bc4-4b71-b617-22a39117fc81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685a23ddaade8 namespace: kserve-ci-e2e-test resourceVersion: "25347" uid: 708f3235-faab-475a-9aa4-a3202801b2f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685a23e54597c namespace: kserve-ci-e2e-test resourceVersion: "25348" uid: a0bff8a2-2646-4c18-b147-6aef70048b54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685a23e5fddee namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: 04079085-99f5-4cf1-a27b-4db7851877ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685a2411bfaad namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: a3c01af6-8c1b-4bb1-b3de-adf15ef31a68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:53:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:54Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685a2419744e2 namespace: kserve-ci-e2e-test resourceVersion: "25352" uid: 68149578-0c7c-4918-a763-d2416ceb5077 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:54:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:54:31Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685a299d79de8 namespace: kserve-ci-e2e-test resourceVersion: "25871" uid: d126bb83-6a40-432d-ba89-a661bd5443f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:55:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:04Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685b280722245 namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: 53a8211e-853a-4d14-96db-695c18fd3290 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:55:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:04Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685b28072c626 namespace: kserve-ci-e2e-test resourceVersion: "26297" uid: 2b406cc9-7b42-459b-ae5b-8fbf632b1c68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:55:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 0b7f1dcc-7061-4a60-ae5e-be54d0b5171e kind: Event lastTimestamp: "2026-07-28T17:55:06Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:55:06Z" name: success-200-isvc-f7704-predictor-865dcf7dff-bjhft.18c685b321fab1c1 namespace: kserve-ci-e2e-test resourceVersion: "26400" uid: bac4cb1f-3b61-44b9-9011-13f84bf36591 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f7704-predictor-865dcf7dff namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 5d08e6a8-8a84-4014-bff0-b814d3cc46c1 kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: 'Created pod: success-200-isvc-f7704-predictor-865dcf7dff-bjhft' metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:53:53Z" name: success-200-isvc-f7704-predictor-865dcf7dff.18c685a21ef33856 namespace: kserve-ci-e2e-test resourceVersion: "25280" uid: c4703683-87f5-40af-a916-c213e98062d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25256" uid: 354e5bc9-247a-43c2-b94e-c648c64820be kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: Scaled up replica set success-200-isvc-f7704-predictor-865dcf7dff from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:53:53Z" name: success-200-isvc-f7704-predictor.18c685a21a814a94 namespace: kserve-ci-e2e-test resourceVersion: "25266" uid: 77f58338-d15b-41ab-b8bb-d261459bf95c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25263" uid: b16c030c-2c52-4990-88f1-bc7089d2b387 kind: Event lastTimestamp: "2026-07-28T17:54:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:08Z" name: success-200-isvc-f7704-predictor.18c685a59934a1dc namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 4a5e1287-c5c7-4d06-85b4-fef31439ca07 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25263" uid: b16c030c-2c52-4990-88f1-bc7089d2b387 kind: Event lastTimestamp: "2026-07-28T17:54:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:08Z" name: success-200-isvc-f7704-predictor.18c685a599aedbcd namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 4aa2a044-bc46-4367-9169-1a2de7e21c7e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:54:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: b16c030c-2c52-4990-88f1-bc7089d2b387 kind: Event lastTimestamp: "2026-07-28T17:54:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:54:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:53Z" name: success-200-isvc-f7704-predictor.18c685a91860c84d namespace: kserve-ci-e2e-test resourceVersion: "26130" uid: ab1c196a-df4c-4085-9793-e2cf078e43a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:54:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7704-predictor namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: b16c030c-2c52-4990-88f1-bc7089d2b387 kind: Event lastTimestamp: "2026-07-28T17:54:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:54:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:54:53Z" name: success-200-isvc-f7704-predictor.18c685a918d3ddd2 namespace: kserve-ci-e2e-test resourceVersion: "26132" uid: b01109d3-2a1e-4aa1-a918-9153726bed5d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7704 namespace: kserve-ci-e2e-test resourceVersion: "25245" uid: ef55a7df-503a-41aa-9759-4ec1aa3ca5c0 kind: Event lastTimestamp: "2026-07-28T17:54:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:54:41Z" name: success-200-isvc-f7704.18c685a21aa18009 namespace: kserve-ci-e2e-test resourceVersion: "26038" uid: a7c8a8de-dbb3-43fe-aa4d-0a0568371f12 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7704 namespace: kserve-ci-e2e-test resourceVersion: "25245" uid: ef55a7df-503a-41aa-9759-4ec1aa3ca5c0 kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: 'Failed to update status for InferenceService "success-200-isvc-f7704": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f7704": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:53:53Z" name: success-200-isvc-f7704.18c685a21b9bf319 namespace: kserve-ci-e2e-test resourceVersion: "25272" uid: 42116dc0-971c-48af-bb78-854e684139a7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7704 namespace: kserve-ci-e2e-test resourceVersion: "25245" uid: ef55a7df-503a-41aa-9759-4ec1aa3ca5c0 kind: Event lastTimestamp: "2026-07-28T17:53:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f7704": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:53:53Z" name: success-200-isvc-f7704.18c685a21b9c69f9 namespace: kserve-ci-e2e-test resourceVersion: "25275" uid: 86c1f86c-018b-4544-9918-67e4e1b9d956 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:54:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7704 namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: ef55a7df-503a-41aa-9759-4ec1aa3ca5c0 kind: Event lastTimestamp: "2026-07-28T17:54:41Z" message: InferenceService [success-200-isvc-f7704] is Ready metadata: creationTimestamp: "2026-07-28T17:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:54:41Z" name: success-200-isvc-f7704.18c685ad5642ba04 namespace: kserve-ci-e2e-test resourceVersion: "26037" uid: 53f280e2-f340-4b2e-ad85-71731d80d49a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:35:13.520466Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-22e0e-5d85974d85-2j87g namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: d84d88cc-b604-4ff6-8534-40919f22c7b5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-22e0e-5d85974d85-2j87g to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:35:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:35:13Z" name: switch-graph-22e0e-5d85974d85-2j87g.18c6849d520fa98f namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: 7f9e575a-b576-4ab6-a604-673f665e3c24 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:13Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-22e0e-5d85974d85-2j87g namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: d84d88cc-b604-4ff6-8534-40919f22c7b5 kind: Event lastTimestamp: "2026-07-28T17:35:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-22e0e-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:13Z" name: switch-graph-22e0e-5d85974d85-2j87g.18c6849d59df1a50 namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 637cf3d6-dee6-429c-9e59-99c4e85bb166 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:14Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-22e0e-5d85974d85-2j87g namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: d84d88cc-b604-4ff6-8534-40919f22c7b5 kind: Event lastTimestamp: "2026-07-28T17:35:14Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:35:14Z" name: switch-graph-22e0e-5d85974d85-2j87g.18c6849d8efd4ae8 namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 814b3417-9877-4680-8d54-7c2d67263aac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-22e0e} kind: Pod name: switch-graph-22e0e-5d85974d85-2j87g namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: d84d88cc-b604-4ff6-8534-40919f22c7b5 kind: Event lastTimestamp: "2026-07-28T17:35:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:14Z" name: switch-graph-22e0e-5d85974d85-2j87g.18c6849d905b7add namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: 3b9c7b9d-8336-4bae-a8f0-980745f1e8a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-22e0e} kind: Pod name: switch-graph-22e0e-5d85974d85-2j87g namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: d84d88cc-b604-4ff6-8534-40919f22c7b5 kind: Event lastTimestamp: "2026-07-28T17:35:14Z" message: 'Created container: switch-graph-22e0e' metadata: creationTimestamp: "2026-07-28T17:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:14Z" name: switch-graph-22e0e-5d85974d85-2j87g.18c6849d93425ed7 namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: 5790c106-3c7a-4f7e-88d8-e3cc91af21c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-22e0e} kind: Pod name: switch-graph-22e0e-5d85974d85-2j87g namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: d84d88cc-b604-4ff6-8534-40919f22c7b5 kind: Event lastTimestamp: "2026-07-28T17:35:14Z" message: Started container switch-graph-22e0e metadata: creationTimestamp: "2026-07-28T17:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:14Z" name: switch-graph-22e0e-5d85974d85-2j87g.18c6849d93b1561d namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: 01f7cdd9-9144-48bd-a65b-65d234a7d75a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-22e0e} kind: Pod name: switch-graph-22e0e-5d85974d85-2j87g namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: d84d88cc-b604-4ff6-8534-40919f22c7b5 kind: Event lastTimestamp: "2026-07-28T17:43:28Z" message: Stopping container switch-graph-22e0e metadata: creationTimestamp: "2026-07-28T17:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:28Z" name: switch-graph-22e0e-5d85974d85-2j87g.18c6851076d4b12c namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: ebcaf020-b12d-4dd8-8a44-530bc9ec7a3f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-22e0e} kind: Pod name: switch-graph-22e0e-5d85974d85-2j87g namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: d84d88cc-b604-4ff6-8534-40919f22c7b5 kind: Event lastTimestamp: "2026-07-28T17:43:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:55Z" name: switch-graph-22e0e-5d85974d85-2j87g.18c6851114f9b490 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: b4770553-75d1-4251-8841-b187d8f2f72a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-22e0e-5d85974d85 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 698476f0-2c79-47ce-9e00-a474ab1b4b2f kind: Event lastTimestamp: "2026-07-28T17:35:13Z" message: 'Created pod: switch-graph-22e0e-5d85974d85-2j87g' metadata: creationTimestamp: "2026-07-28T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:13Z" name: switch-graph-22e0e-5d85974d85.18c6849d5179675a namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: a6690021-e55e-475d-a08f-5c31f6902cfc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-22e0e namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: 9f7469d6-e8b1-4a02-9b6e-2e33847da4b2 kind: Event lastTimestamp: "2026-07-28T17:35:13Z" message: Scaled up replica set switch-graph-22e0e-5d85974d85 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:13Z" name: switch-graph-22e0e.18c6849d50a29a63 namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: ad6b4fae-938e-4edb-9876-7983acf4dfba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-22e0e namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 26f6428b-50f6-4d4e-825b-44464f3e7304 kind: Event lastTimestamp: "2026-07-28T17:35:13Z" message: InferenceGraph [switch-graph-22e0e] is Ready metadata: creationTimestamp: "2026-07-28T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:13Z" name: switch-graph-22e0e.18c6849d562dd211 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: b265038c-42b9-4d1e-aaf9-37f0b1f0b198 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-22e0e namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 26f6428b-50f6-4d4e-825b-44464f3e7304 kind: Event lastTimestamp: "2026-07-28T17:35:13Z" message: 'Failed to update status for InferenceGraph "switch-graph-22e0e": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-22e0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:13Z" name: switch-graph-22e0e.18c6849d58f9261b namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: e53782bd-5727-44fd-9a36-b17e72ab5a66 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-22e0e namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 26f6428b-50f6-4d4e-825b-44464f3e7304 kind: Event lastTimestamp: "2026-07-28T17:35:13Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-22e0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:13Z" name: switch-graph-22e0e.18c6849d58f98742 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 3f18f70f-9b6c-410b-9244-9f9ae4fc6452 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:35:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-22e0e namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: 59e884ac-595c-4096-8605-938b622555f8 kind: Event lastTimestamp: "2026-07-28T17:35:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:58Z" name: switch-graph-22e0e.18c684a0cf1a3154 namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 7940c452-ef56-4f2a-96c0-95682127889e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:35:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-22e0e namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: 59e884ac-595c-4096-8605-938b622555f8 kind: Event lastTimestamp: "2026-07-28T17:35:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:58Z" name: switch-graph-22e0e.18c684a0cf878caa namespace: kserve-ci-e2e-test resourceVersion: "16825" uid: 3a43e5c4-5450-417a-ac8f-f4f720e07930 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:56:17.544925Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cb1ee-7ddbb59689-cbvqc namespace: kserve-ci-e2e-test resourceVersion: "27218" uid: 0792112d-477d-469c-a9bf-10b4d52ec4ab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-cb1ee-7ddbb59689-cbvqc to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:56:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:56:17Z" name: switch-graph-cb1ee-7ddbb59689-cbvqc.18c685c39fca4a96 namespace: kserve-ci-e2e-test resourceVersion: "27222" uid: 4e5b4471-c8d1-4e79-ad0a-e0fe77f16294 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:17Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cb1ee-7ddbb59689-cbvqc namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 0792112d-477d-469c-a9bf-10b4d52ec4ab kind: Event lastTimestamp: "2026-07-28T17:56:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-cb1ee-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:56:17Z" name: switch-graph-cb1ee-7ddbb59689-cbvqc.18c685c3a91ab09b namespace: kserve-ci-e2e-test resourceVersion: "27231" uid: a47a2b3d-1be3-448c-a4b0-359362ded8bf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:18Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cb1ee-7ddbb59689-cbvqc namespace: kserve-ci-e2e-test resourceVersion: "27228" uid: 0792112d-477d-469c-a9bf-10b4d52ec4ab kind: Event lastTimestamp: "2026-07-28T17:56:18Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:56:18Z" name: switch-graph-cb1ee-7ddbb59689-cbvqc.18c685c3dcf6c2a8 namespace: kserve-ci-e2e-test resourceVersion: "27246" uid: 42e271ca-7b2b-42b4-ab80-cec874a639d8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb1ee} kind: Pod name: switch-graph-cb1ee-7ddbb59689-cbvqc namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 0792112d-477d-469c-a9bf-10b4d52ec4ab kind: Event lastTimestamp: "2026-07-28T17:56:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:56:18Z" name: switch-graph-cb1ee-7ddbb59689-cbvqc.18c685c3deea73c7 namespace: kserve-ci-e2e-test resourceVersion: "27248" uid: e33f88ee-37ba-4fc0-9811-0d68fc06a7c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb1ee} kind: Pod name: switch-graph-cb1ee-7ddbb59689-cbvqc namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 0792112d-477d-469c-a9bf-10b4d52ec4ab kind: Event lastTimestamp: "2026-07-28T17:56:18Z" message: 'Created container: switch-graph-cb1ee' metadata: creationTimestamp: "2026-07-28T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:56:18Z" name: switch-graph-cb1ee-7ddbb59689-cbvqc.18c685c3e26bcd1e namespace: kserve-ci-e2e-test resourceVersion: "27249" uid: a521a141-5ad1-43a0-b8b4-b6181ebc682f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb1ee} kind: Pod name: switch-graph-cb1ee-7ddbb59689-cbvqc namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 0792112d-477d-469c-a9bf-10b4d52ec4ab kind: Event lastTimestamp: "2026-07-28T17:56:18Z" message: Started container switch-graph-cb1ee metadata: creationTimestamp: "2026-07-28T17:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:56:18Z" name: switch-graph-cb1ee-7ddbb59689-cbvqc.18c685c3e2e6d2d1 namespace: kserve-ci-e2e-test resourceVersion: "27251" uid: 7130cdba-901c-4c32-a044-3013d77b0b7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb1ee} kind: Pod name: switch-graph-cb1ee-7ddbb59689-cbvqc namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 0792112d-477d-469c-a9bf-10b4d52ec4ab kind: Event lastTimestamp: "2026-07-28T18:12:36Z" message: Stopping container switch-graph-cb1ee metadata: creationTimestamp: "2026-07-28T18:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:12:36Z" name: switch-graph-cb1ee-7ddbb59689-cbvqc.18c686a7956a96f3 namespace: kserve-ci-e2e-test resourceVersion: "34039" uid: f9677612-9097-417f-afe9-c177bbc65756 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T18:12:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb1ee} kind: Pod name: switch-graph-cb1ee-7ddbb59689-cbvqc namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 0792112d-477d-469c-a9bf-10b4d52ec4ab kind: Event lastTimestamp: "2026-07-28T18:13:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T18:12:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:13:05Z" name: switch-graph-cb1ee-7ddbb59689-cbvqc.18c686a86fd76457 namespace: kserve-ci-e2e-test resourceVersion: "34316" uid: d846da24-f650-4502-aec2-358139d1c049 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-cb1ee-7ddbb59689 namespace: kserve-ci-e2e-test resourceVersion: "27215" uid: 21d6a604-baa0-4dbb-8606-347feacae3a5 kind: Event lastTimestamp: "2026-07-28T17:56:17Z" message: 'Created pod: switch-graph-cb1ee-7ddbb59689-cbvqc' metadata: creationTimestamp: "2026-07-28T17:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:17Z" name: switch-graph-cb1ee-7ddbb59689.18c685c39e50f605 namespace: kserve-ci-e2e-test resourceVersion: "27221" uid: e3b05446-e577-4d61-a099-b62a16c6d158 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "27213" uid: 1508f031-730d-46bf-9c78-d534e0f4e589 kind: Event lastTimestamp: "2026-07-28T17:56:17Z" message: Scaled up replica set switch-graph-cb1ee-7ddbb59689 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:17Z" name: switch-graph-cb1ee.18c685c39c283b32 namespace: kserve-ci-e2e-test resourceVersion: "27217" uid: c194ffc1-e473-48e1-a6c5-5c96b7ea8965 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "27234" uid: 4992c982-5ed4-44c9-8ba2-7d75f7dd1bc1 kind: Event lastTimestamp: "2026-07-28T17:56:17Z" message: InferenceGraph [switch-graph-cb1ee] is Ready metadata: creationTimestamp: "2026-07-28T17:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:56:17Z" name: switch-graph-cb1ee.18c685c3ae67c272 namespace: kserve-ci-e2e-test resourceVersion: "27236" uid: a811256a-00a0-43d3-a9f9-f0f657f49e87 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: 86b268ae-94ed-4b70-b22d-425036be7b2b kind: Event lastTimestamp: "2026-07-28T17:56:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:32Z" name: switch-graph-cb1ee.18c685c71b41de04 namespace: kserve-ci-e2e-test resourceVersion: "27363" uid: a3998396-bf40-4b01-ad5a-91fb517d328e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: 86b268ae-94ed-4b70-b22d-425036be7b2b kind: Event lastTimestamp: "2026-07-28T17:56:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:32Z" name: switch-graph-cb1ee.18c685c71bc37e8a namespace: kserve-ci-e2e-test resourceVersion: "27365" uid: 9b80bd81-df41-45fe-b4a8-49baaf9b1318 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "27364" uid: 86b268ae-94ed-4b70-b22d-425036be7b2b kind: Event lastTimestamp: "2026-07-28T17:56:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:47Z" name: switch-graph-cb1ee.18c685ca9a9def9c namespace: kserve-ci-e2e-test resourceVersion: "27475" uid: 9724e81f-6b93-48fd-80ff-e9ebb7cde11d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-cb1ee namespace: kserve-ci-e2e-test resourceVersion: "27364" uid: 86b268ae-94ed-4b70-b22d-425036be7b2b kind: Event lastTimestamp: "2026-07-28T17:56:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:56:47Z" name: switch-graph-cb1ee.18c685ca9b0d932c namespace: kserve-ci-e2e-test resourceVersion: "27477" uid: 3e9bf4ad-86c7-4fca-8608-b0339353feb6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:59.102848Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f0903-56678cb99b-4nwlp namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 484ebf5f-3805-45f2-97a1-f3cb4caec5dd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f0903-56678cb99b-4nwlp to ip-10-0-131-233.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:59Z" name: switch-graph-f0903-56678cb99b-4nwlp.18c6848bfe6cf15f namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5973ab93-4a85-4f2c-b5fb-71b22be3d3e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76477f4c5d-md67n source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:59Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f0903-56678cb99b-4nwlp namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 484ebf5f-3805-45f2-97a1-f3cb4caec5dd kind: Event lastTimestamp: "2026-07-28T17:33:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-f0903-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:33:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:59Z" name: switch-graph-f0903-56678cb99b-4nwlp.18c6848c0aff5d3f namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 25dd9577-4030-4fd0-92d6-0d17f708e951 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:00Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f0903-56678cb99b-4nwlp namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 484ebf5f-3805-45f2-97a1-f3cb4caec5dd kind: Event lastTimestamp: "2026-07-28T17:34:00Z" message: Add eth0 [10.133.0.15/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:34:00Z" name: switch-graph-f0903-56678cb99b-4nwlp.18c6848c3b305ad5 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 280c5b9f-1299-4134-9072-babc24a28a9c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0903} kind: Pod name: switch-graph-f0903-56678cb99b-4nwlp namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 484ebf5f-3805-45f2-97a1-f3cb4caec5dd kind: Event lastTimestamp: "2026-07-28T17:34:00Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" metadata: creationTimestamp: "2026-07-28T17:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:00Z" name: switch-graph-f0903-56678cb99b-4nwlp.18c6848c3c82eed6 namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: ba16b303-2adb-475c-8b51-7a326a397f65 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0903} kind: Pod name: switch-graph-f0903-56678cb99b-4nwlp namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 484ebf5f-3805-45f2-97a1-f3cb4caec5dd kind: Event lastTimestamp: "2026-07-28T17:34:02Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" in 2.343s (2.343s including waiting). Image size: 218226779 bytes.' metadata: creationTimestamp: "2026-07-28T17:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:02Z" name: switch-graph-f0903-56678cb99b-4nwlp.18c6848cc837fbad namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 89b81348-046d-4639-9354-03c3dfd2fa3e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0903} kind: Pod name: switch-graph-f0903-56678cb99b-4nwlp namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 484ebf5f-3805-45f2-97a1-f3cb4caec5dd kind: Event lastTimestamp: "2026-07-28T17:34:02Z" message: 'Created container: switch-graph-f0903' metadata: creationTimestamp: "2026-07-28T17:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:02Z" name: switch-graph-f0903-56678cb99b-4nwlp.18c6848ccb362c6f namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: 9b4e5b79-286d-41a3-8025-c493eea4204c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0903} kind: Pod name: switch-graph-f0903-56678cb99b-4nwlp namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 484ebf5f-3805-45f2-97a1-f3cb4caec5dd kind: Event lastTimestamp: "2026-07-28T17:34:02Z" message: Started container switch-graph-f0903 metadata: creationTimestamp: "2026-07-28T17:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:02Z" name: switch-graph-f0903-56678cb99b-4nwlp.18c6848ccba9cb4d namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 051f007e-1e75-4f97-bca0-9a112fc8d7f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0903} kind: Pod name: switch-graph-f0903-56678cb99b-4nwlp namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 484ebf5f-3805-45f2-97a1-f3cb4caec5dd kind: Event lastTimestamp: "2026-07-28T17:34:13Z" message: Stopping container switch-graph-f0903 metadata: creationTimestamp: "2026-07-28T17:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:13Z" name: switch-graph-f0903-56678cb99b-4nwlp.18c6848f4b350367 namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: cb21cf75-f53d-4e58-a443-8fcdcf6f81e4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f0903} kind: Pod name: switch-graph-f0903-56678cb99b-4nwlp namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 484ebf5f-3805-45f2-97a1-f3cb4caec5dd kind: Event lastTimestamp: "2026-07-28T17:34:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:39Z" name: switch-graph-f0903-56678cb99b-4nwlp.18c6848f9255fa50 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: f07aa8e5-0492-49c9-b38f-12dc8e3bcb91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-233.ec2.internal source: component: kubelet host: ip-10-0-131-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f0903-56678cb99b namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 6f5d74a5-9616-4c5f-99c9-e2e215c56ef6 kind: Event lastTimestamp: "2026-07-28T17:33:59Z" message: 'Created pod: switch-graph-f0903-56678cb99b-4nwlp' metadata: creationTimestamp: "2026-07-28T17:33:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:59Z" name: switch-graph-f0903-56678cb99b.18c6848bfdd93a64 namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 6d6a4457-02a7-4230-ac73-7601cbfdfb04 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f0903 namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 6c9f42f7-7470-4a83-bbdf-2a094ce7c4a5 kind: Event lastTimestamp: "2026-07-28T17:33:59Z" message: Scaled up replica set switch-graph-f0903-56678cb99b from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:59Z" name: switch-graph-f0903.18c6848bfc4c615c namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: dc63c489-97bc-4090-866d-99b175f71794 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f0903 namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 6f928f97-d35a-4bac-bc64-c91e6736e4f9 kind: Event lastTimestamp: "2026-07-28T17:33:59Z" message: InferenceGraph [switch-graph-f0903] is Ready metadata: creationTimestamp: "2026-07-28T17:33:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:59Z" name: switch-graph-f0903.18c6848c07c64632 namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: b18e1b7b-3247-4cd7-8e83-4ddfd2e470eb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "34327"