--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:56:17.772731Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-42ebc-7bff6d66c7-qkhvz namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: 77961d8f-cadf-4169-b98c-5ebda502652f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-42ebc-7bff6d66c7-qkhvz to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:56:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:56:17Z" name: ensemble-graph-42ebc-7bff6d66c7-qkhvz.18c0968b5ebc621e namespace: kserve-ci-e2e-test resourceVersion: "21607" uid: 5486ef08-0ac2-4409-b6a8-72be88190880 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:18Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-42ebc-7bff6d66c7-qkhvz namespace: kserve-ci-e2e-test resourceVersion: "21614" uid: 77961d8f-cadf-4169-b98c-5ebda502652f kind: Event lastTimestamp: "2026-07-09T09:56:18Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:56:18Z" name: ensemble-graph-42ebc-7bff6d66c7-qkhvz.18c0968b7826f88a namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: c0b71b51-1832-4522-abe8-9cbaedef5912 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-42ebc} kind: Pod name: ensemble-graph-42ebc-7bff6d66c7-qkhvz namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 77961d8f-cadf-4169-b98c-5ebda502652f kind: Event lastTimestamp: "2026-07-09T09:56:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T09:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:18Z" name: ensemble-graph-42ebc-7bff6d66c7-qkhvz.18c0968b7990fd52 namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: f6fd805d-72b8-4c9f-b13a-1fec2fc7a2ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-42ebc} kind: Pod name: ensemble-graph-42ebc-7bff6d66c7-qkhvz namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 77961d8f-cadf-4169-b98c-5ebda502652f kind: Event lastTimestamp: "2026-07-09T09:56:18Z" message: 'Created container: ensemble-graph-42ebc' metadata: creationTimestamp: "2026-07-09T09:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:18Z" name: ensemble-graph-42ebc-7bff6d66c7-qkhvz.18c0968b7cac1b71 namespace: kserve-ci-e2e-test resourceVersion: "21632" uid: 22176294-b7ef-4821-92bf-c0813db5c1d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-42ebc} kind: Pod name: ensemble-graph-42ebc-7bff6d66c7-qkhvz namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 77961d8f-cadf-4169-b98c-5ebda502652f kind: Event lastTimestamp: "2026-07-09T09:56:18Z" message: Started container ensemble-graph-42ebc metadata: creationTimestamp: "2026-07-09T09:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:18Z" name: ensemble-graph-42ebc-7bff6d66c7-qkhvz.18c0968b7d1ed592 namespace: kserve-ci-e2e-test resourceVersion: "21633" uid: 568adeec-f12f-4cb3-9032-ac717dba5247 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-42ebc} kind: Pod name: ensemble-graph-42ebc-7bff6d66c7-qkhvz namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 77961d8f-cadf-4169-b98c-5ebda502652f kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: Stopping container ensemble-graph-42ebc metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:32Z" name: ensemble-graph-42ebc-7bff6d66c7-qkhvz.18c096fe87beda8c namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: d158188f-4703-4d9c-be03-3e2b0d6525c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:04:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-42ebc} kind: Pod name: ensemble-graph-42ebc-7bff6d66c7-qkhvz namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 77961d8f-cadf-4169-b98c-5ebda502652f kind: Event lastTimestamp: "2026-07-09T10:04:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T10:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:59Z" name: ensemble-graph-42ebc-7bff6d66c7-qkhvz.18c096ff0e7b7b2e namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: 681cfc4b-6cba-4d7e-8687-eaf861de6d58 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-42ebc-7bff6d66c7 namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: 97fb8a4f-2faf-4750-921e-69d0c31e4830 kind: Event lastTimestamp: "2026-07-09T09:56:17Z" message: 'Created pod: ensemble-graph-42ebc-7bff6d66c7-qkhvz' metadata: creationTimestamp: "2026-07-09T09:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:17Z" name: ensemble-graph-42ebc-7bff6d66c7.18c0968b5e4fdbf9 namespace: kserve-ci-e2e-test resourceVersion: "21605" uid: 6858a31d-db66-4c9e-afba-b60dca16b236 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-42ebc namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: a657e15b-0fb8-4e32-9d6b-2ff3ae5ea1ef kind: Event lastTimestamp: "2026-07-09T09:56:17Z" message: Scaled up replica set ensemble-graph-42ebc-7bff6d66c7 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:17Z" name: ensemble-graph-42ebc.18c0968b5d8b3948 namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 856f2db8-62ce-4759-9f51-4e02e16337d2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-42ebc namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: 48c83837-f62d-4a50-abe4-9db0fc4ce3aa kind: Event lastTimestamp: "2026-07-09T09:56:17Z" message: InferenceGraph [ensemble-graph-42ebc] is Ready metadata: creationTimestamp: "2026-07-09T09:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:17Z" name: ensemble-graph-42ebc.18c0968b63b1e184 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: 2c5c0157-9b07-4329-9c65-ab6c6b19e16c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-42ebc namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: 48c83837-f62d-4a50-abe4-9db0fc4ce3aa kind: Event lastTimestamp: "2026-07-09T09:56:17Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-42ebc": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-42ebc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:17Z" name: ensemble-graph-42ebc.18c0968b6a51cb02 namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: 04fe5616-a584-451f-a0e8-47ded845f967 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-42ebc namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: 48c83837-f62d-4a50-abe4-9db0fc4ce3aa kind: Event lastTimestamp: "2026-07-09T09:56:17Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-42ebc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:17Z" name: ensemble-graph-42ebc.18c0968b6a522d4b namespace: kserve-ci-e2e-test resourceVersion: "21624" uid: c7994724-d666-4d10-97d3-11b53b3bb053 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:56:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-42ebc namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 69dd76d9-4de4-4a56-9359-62826959efec kind: Event lastTimestamp: "2026-07-09T09:57:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:02Z" name: ensemble-graph-42ebc.18c0968edbf9b1e0 namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: 759157bc-1689-4f79-8e6c-1951e8ae9895 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:56:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-42ebc namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 69dd76d9-4de4-4a56-9359-62826959efec kind: Event lastTimestamp: "2026-07-09T09:57:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:02Z" name: ensemble-graph-42ebc.18c0968edc4372c7 namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: 9c2eb571-7284-4dc8-8acd-202226caaee3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:55:07.440259Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b04db-d79cb4dd-kpjls namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: 285e714e-87ec-43b3-b83c-f53fb2d7dca7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-b04db-d79cb4dd-kpjls to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:55:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:55:07Z" name: ensemble-graph-b04db-d79cb4dd-kpjls.18c0967afe97fa5b namespace: kserve-ci-e2e-test resourceVersion: "20638" uid: 26c24fd3-099a-4dbb-b5b1-012c86e6dbf3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:07Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b04db-d79cb4dd-kpjls namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 285e714e-87ec-43b3-b83c-f53fb2d7dca7 kind: Event lastTimestamp: "2026-07-09T09:55:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-b04db-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:07Z" name: ensemble-graph-b04db-d79cb4dd-kpjls.18c0967b0e6783ff namespace: kserve-ci-e2e-test resourceVersion: "20650" uid: b88a8ef2-fe8b-480c-b380-a2c31fece0ac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:08Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b04db-d79cb4dd-kpjls namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 285e714e-87ec-43b3-b83c-f53fb2d7dca7 kind: Event lastTimestamp: "2026-07-09T09:55:08Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:55:08Z" name: ensemble-graph-b04db-d79cb4dd-kpjls.18c0967b3bd30f17 namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: 6d425d1f-6283-441d-bbb1-e89b3cb85926 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b04db} kind: Pod name: ensemble-graph-b04db-d79cb4dd-kpjls namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 285e714e-87ec-43b3-b83c-f53fb2d7dca7 kind: Event lastTimestamp: "2026-07-09T09:55:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T09:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:08Z" name: ensemble-graph-b04db-d79cb4dd-kpjls.18c0967b3d29a3c5 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 1244e694-7c48-450b-843a-2456bc1cde19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b04db} kind: Pod name: ensemble-graph-b04db-d79cb4dd-kpjls namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 285e714e-87ec-43b3-b83c-f53fb2d7dca7 kind: Event lastTimestamp: "2026-07-09T09:55:08Z" message: 'Created container: ensemble-graph-b04db' metadata: creationTimestamp: "2026-07-09T09:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:08Z" name: ensemble-graph-b04db-d79cb4dd-kpjls.18c0967b408c7d60 namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: ab326a00-db17-451d-9c1e-f8d9053fdb06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b04db} kind: Pod name: ensemble-graph-b04db-d79cb4dd-kpjls namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 285e714e-87ec-43b3-b83c-f53fb2d7dca7 kind: Event lastTimestamp: "2026-07-09T09:55:08Z" message: Started container ensemble-graph-b04db metadata: creationTimestamp: "2026-07-09T09:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:08Z" name: ensemble-graph-b04db-d79cb4dd-kpjls.18c0967b40f9252f namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: 14dea3ab-ee36-442c-965d-23f6a45b71d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b04db} kind: Pod name: ensemble-graph-b04db-d79cb4dd-kpjls namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 285e714e-87ec-43b3-b83c-f53fb2d7dca7 kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: Stopping container ensemble-graph-b04db metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:17Z" name: ensemble-graph-b04db-d79cb4dd-kpjls.18c0967d575317ce namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 9140f9b2-0c70-46fc-83f1-42954b6dd3d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:55:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b04db} kind: Pod name: ensemble-graph-b04db-d79cb4dd-kpjls namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 285e714e-87ec-43b3-b83c-f53fb2d7dca7 kind: Event lastTimestamp: "2026-07-09T09:55:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T09:55:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:45Z" name: ensemble-graph-b04db-d79cb4dd-kpjls.18c0967dfdf2cd7e namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 9136454d-3795-45f5-a1a1-39219fafc485 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-b04db-d79cb4dd namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: 0640d4fa-0bf3-4505-aa8f-c94ec965f643 kind: Event lastTimestamp: "2026-07-09T09:55:07Z" message: 'Created pod: ensemble-graph-b04db-d79cb4dd-kpjls' metadata: creationTimestamp: "2026-07-09T09:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:07Z" name: ensemble-graph-b04db-d79cb4dd.18c0967afe0dadfb namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: 3caa9940-24de-4ff7-94f2-970f315358d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-b04db namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 4812200d-e56c-4f90-9778-181b686d5859 kind: Event lastTimestamp: "2026-07-09T09:55:07Z" message: Scaled up replica set ensemble-graph-b04db-d79cb4dd from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:07Z" name: ensemble-graph-b04db.18c0967afd5febf4 namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 0bbc227b-9a76-42c3-af2a-b72d71bc2db9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-b04db namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 466b1989-0e32-488a-b513-39348f3ac0a3 kind: Event lastTimestamp: "2026-07-09T09:55:07Z" message: InferenceGraph [ensemble-graph-b04db] is Ready metadata: creationTimestamp: "2026-07-09T09:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:55:07Z" name: ensemble-graph-b04db.18c0967b05418e80 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 5c9fd1ea-ad8b-4f79-b34b-6ef40f6a5255 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:43:18.546499Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:43:18Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d5f13ba717 namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: c0c9abf4-81c6-40e2-976d-bfe14bc0085c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1980c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:18Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d600a3eb1b namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 30ec09d7-e802-4e21-acc8-56a4ffff70d6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14465" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:43:19Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d62eab5161 namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: 6f497839-0389-4024-8c55-5e1a55216bc2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d630139721 namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 55404bc8-e769-42aa-a483-d0f4626a5680 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:32Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" in 12.843s (12.843s including waiting). Image size: 1334626399 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:32Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d92d975556 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 7ba95f3b-99e7-44a1-8063-f15f7338eed3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:32Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d9309ff39e namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: 1dad35e2-64d6-4776-af39-0789358098d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:32Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d93112422a namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: c19a7089-7d5a-40c5-b743-bed7192a9a90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T09:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:32Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d9311e3bec namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 4bde50b4-a2e1-42e9-a3b7-7feea76a6763 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:34Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.34s (2.34s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:34Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d9bca3a2ef namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 8c85ad9d-0dd5-4c04-a31d-908082f1265f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:34Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d9c03868e0 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 6455109c-e8f3-4ed8-9d65-5a92a140f600 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:43:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:34Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095d9c0a931cf namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 84574f5a-40e3-4a0f-8389-339fb7f3f57f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:44:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:12Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095da0ea2b328 namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 23b53983-a007-47bd-a323-53e848696239 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:52Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095ebd8e8c81f namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: 866f5f56-d9b1-4a45-a1a9-d0c080b13eb4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: cc6cecf0-dc4e-42a0-a988-d77b3bd974ee kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:52Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf.18c095ebd8e95141 namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: b1c4915c-7f5d-40bd-a8d2-24959f41b35f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1980c-predictor-5ff6dfb79c namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 73fd4ae1-e7fe-48c2-9134-5682b66bc95d kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'Created pod: error-404-isvc-1980c-predictor-5ff6dfb79c-x64tf' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: error-404-isvc-1980c-predictor-5ff6dfb79c.18c095d5f0dbe79c namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: 2036d5b8-994f-491b-800c-f3b2e37bc0e8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: c9557a69-6a08-4657-96e6-f50c7d8b6eed kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: Scaled up replica set error-404-isvc-1980c-predictor-5ff6dfb79c from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: error-404-isvc-1980c-predictor.18c095d5ef3a7f9f namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: 0ef3c680-786f-4c91-aeff-742db78ef73b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: 45cfdc6f-80ce-429a-8b5a-7344995c859f kind: Event lastTimestamp: "2026-07-09T09:43:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:48Z" name: error-404-isvc-1980c-predictor.18c095d96db0653f namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 6fe9d20c-1ddf-4bb7-ac4f-72d42f2258a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: 45cfdc6f-80ce-429a-8b5a-7344995c859f kind: Event lastTimestamp: "2026-07-09T09:43:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:48Z" name: error-404-isvc-1980c-predictor.18c095d96e200094 namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 5e0bd46d-155e-4d75-aba2-7b084f874ad4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:44:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: 45cfdc6f-80ce-429a-8b5a-7344995c859f kind: Event lastTimestamp: "2026-07-09T09:44:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:48Z" name: error-404-isvc-1980c-predictor.18c095e06bc3b858 namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: f0ebe99b-c22c-47fe-8431-0ef9a0e58e4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:44:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: 45cfdc6f-80ce-429a-8b5a-7344995c859f kind: Event lastTimestamp: "2026-07-09T09:44:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:48Z" name: error-404-isvc-1980c-predictor.18c095e06c3620d1 namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 9bcfad54-816a-4ec3-9124-7b37a9efef2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1980c namespace: kserve-ci-e2e-test resourceVersion: "14432" uid: 954440e2-6ba5-4dec-9a1f-e58178300f2e kind: Event lastTimestamp: "2026-07-09T09:44:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:22Z" name: error-404-isvc-1980c.18c095d5ef3fded1 namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 30fae2d5-1d3c-4b02-8884-b8ad48aa5cc1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1980c namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 954440e2-6ba5-4dec-9a1f-e58178300f2e kind: Event lastTimestamp: "2026-07-09T09:44:22Z" message: InferenceService [error-404-isvc-1980c] is Ready metadata: creationTimestamp: "2026-07-09T09:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:22Z" name: error-404-isvc-1980c.18c095e4c9f09b35 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 4cb030f6-d3d2-4612-a87b-07c3322d6e72 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:56:13.680095Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:56:13Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0968a6acbb643 namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: 41090613-d55d-402b-8342-244c2a6865ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:56:14Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0968a84333f33 namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: 76ef414a-592b-4f16-ad0e-23dc70aca4ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0968a85e147e6 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: 51780cf8-9b69-4f95-a651-b2f659a2e499 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0968a88aa69df namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 72dadf01-ec73-49d1-a83c-83242b0fcc89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0968a895fba73 namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: c36c8092-469a-42d0-80ea-ff5f967070f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0968a89685455 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 3cdb176f-7c93-4047-af38-bab2a97873ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0968a8c0529a5 namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 00a1dc5d-5d61-4582-ba82-cc78bd853df5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0968a8c739720 namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: cb21548d-6aad-42a6-99b1-9ef6356a7d6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T09:56:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:51Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0968ade2b1573 namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: f3bacd53-b3ce-48c3-83b1-b5801668e5a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:28Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0970b963de160 namespace: kserve-ci-e2e-test resourceVersion: "26020" uid: d1b604a6-a2cb-4ba7-a08e-07cd99c7cf2f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: bbaad733-fd9e-4540-a67a-0e37f9a91984 kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:28Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85.18c0970b963e82ba namespace: kserve-ci-e2e-test resourceVersion: "26022" uid: 172f2e82-0d5f-44f1-9c14-bd3a125b29f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2cd6b-predictor-7d4869ff97 namespace: kserve-ci-e2e-test resourceVersion: "21495" uid: e154c702-19dc-4e31-96b6-3d63b2ac13a3 kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: 'Created pod: error-404-isvc-2cd6b-predictor-7d4869ff97-jqr85' metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:13Z" name: error-404-isvc-2cd6b-predictor-7d4869ff97.18c0968a6a5fce53 namespace: kserve-ci-e2e-test resourceVersion: "21503" uid: 1dddf6ac-f089-4aa9-ac6c-9de6343636d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: b10c8e9e-d447-446c-b3f5-92d41e9a28d0 kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: Scaled up replica set error-404-isvc-2cd6b-predictor-7d4869ff97 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:13Z" name: error-404-isvc-2cd6b-predictor.18c0968a689c941f namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: dc894014-470c-4840-82e2-4aeadf15a84f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: b921b8f1-5aea-4458-9307-734484a39cee kind: Event lastTimestamp: "2026-07-09T09:56:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:28Z" name: error-404-isvc-2cd6b-predictor.18c0968de717a64c namespace: kserve-ci-e2e-test resourceVersion: "21718" uid: 2f649969-82ca-4284-9ea8-457f98e26b7c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: b921b8f1-5aea-4458-9307-734484a39cee kind: Event lastTimestamp: "2026-07-09T09:56:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:28Z" name: error-404-isvc-2cd6b-predictor.18c0968de76903e2 namespace: kserve-ci-e2e-test resourceVersion: "21720" uid: 8198311d-22b9-45d7-ad2a-612fe12592b1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:56:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: b921b8f1-5aea-4458-9307-734484a39cee kind: Event lastTimestamp: "2026-07-09T09:57:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:28Z" name: error-404-isvc-2cd6b-predictor.18c09691661c33f3 namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: e5cbffd3-3f7c-4ed8-b981-649cdb7c8bf4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:56:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: b921b8f1-5aea-4458-9307-734484a39cee kind: Event lastTimestamp: "2026-07-09T09:57:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:28Z" name: error-404-isvc-2cd6b-predictor.18c0969166866ea9 namespace: kserve-ci-e2e-test resourceVersion: "22236" uid: cd09d70b-c7b7-4d09-bbcf-777a1f5dbba1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: a7aa29a1-b5fc-4596-a520-ea163c9dd70d kind: Event lastTimestamp: "2026-07-09T09:57:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:57:01Z" name: error-404-isvc-2cd6b.18c0968a68a42ed5 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 1b80aedd-ae60-4036-94ff-53c558a572d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: a7aa29a1-b5fc-4596-a520-ea163c9dd70d kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: 'Failed to update status for InferenceService "error-404-isvc-2cd6b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2cd6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:13Z" name: error-404-isvc-2cd6b.18c0968a690747f6 namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: edc52348-8891-4b91-9709-e6db92191a51 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: a7aa29a1-b5fc-4596-a520-ea163c9dd70d kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2cd6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:13Z" name: error-404-isvc-2cd6b.18c0968a690779ba namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 4f6f8c23-6c69-422d-a80d-9b07081f6109 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:57:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: a7aa29a1-b5fc-4596-a520-ea163c9dd70d kind: Event lastTimestamp: "2026-07-09T09:57:01Z" message: InferenceService [error-404-isvc-2cd6b] is Ready metadata: creationTimestamp: "2026-07-09T09:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:57:01Z" name: error-404-isvc-2cd6b.18c0969599aef8ed namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: 9faa07f7-ca69-4084-b5b7-f6d3dc559fdd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:55:17.795340Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:55:17Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c0967d67cdfe5f namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: d59ae420-424a-4430-bc72-cce1052ef6b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20843" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:55:18Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c0967d813ed2e9 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: fa36794a-d648-4d94-b3dd-cb3d86a1bbc8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c0967d829286eb namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 3090d9b3-b889-4ee7-a57a-fe83acdc95e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c0967d85a0e1f4 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: 66c1b15c-8fce-4bb1-8793-20b70ccbcab0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c0967d8610a63d namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: 7b30a6f5-cdb9-4e78-99a6-889affa70ccb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c0967d861a6adc namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: fad95580-77e1-4b34-b045-486081a984b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c0967d8906c186 namespace: kserve-ci-e2e-test resourceVersion: "20869" uid: d5a5f64a-6710-4ac6-9e93-3ea73bcbd0a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c0967d8976e90a namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: 7cdfda33-2dd2-494b-8134-ffb064306a67 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T09:55:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T10:04:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:35Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c0967dc4cc2480 namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: 38c91274-1cfc-4e32-8609-15999a9c40dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:32Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c096fe95d11333 namespace: kserve-ci-e2e-test resourceVersion: "25254" uid: 7382e3cd-1b11-4333-b590-94eeabf5ecd3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:32Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c096fe95d19102 namespace: kserve-ci-e2e-test resourceVersion: "25257" uid: c41d27f1-9683-4b2a-8c93-cf78a3f02ba5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 98131af6-44e2-42e6-a79c-98e85df05f66 kind: Event lastTimestamp: "2026-07-09T10:04:35Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:35Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w.18c096ff39382b37 namespace: kserve-ci-e2e-test resourceVersion: "25407" uid: e2a40bdb-f522-48be-8364-6b5feca73654 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-42ebc-predictor-5b56bdc44c namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 3d09ef54-9c77-4e49-bf72-1359c841b33e kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: 'Created pod: error-404-isvc-42ebc-predictor-5b56bdc44c-hwc6w' metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:17Z" name: error-404-isvc-42ebc-predictor-5b56bdc44c.18c0967d675d9dd4 namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: b1b0244e-bc50-43a6-8f05-18d9caa8a32c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 4d328f2d-cbd0-4470-826e-82e82272d6bd kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: Scaled up replica set error-404-isvc-42ebc-predictor-5b56bdc44c from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:17Z" name: error-404-isvc-42ebc-predictor.18c0967d65b14d80 namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: d95fac3f-5a42-474e-b0e2-eeae54f64494 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: dc37ed2f-0e43-48ba-9bde-c77a19d35975 kind: Event lastTimestamp: "2026-07-09T09:55:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:32Z" name: error-404-isvc-42ebc-predictor.18c09680e408c633 namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: ce236fe0-c6b4-4b97-9a12-bd1f162aaa97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: dc37ed2f-0e43-48ba-9bde-c77a19d35975 kind: Event lastTimestamp: "2026-07-09T09:55:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:32Z" name: error-404-isvc-42ebc-predictor.18c09680e472d3fd namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: a574c54a-35f5-4799-a8ef-b87cee957d76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:55:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: dc37ed2f-0e43-48ba-9bde-c77a19d35975 kind: Event lastTimestamp: "2026-07-09T09:56:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:47Z" name: error-404-isvc-42ebc-predictor.18c09684631a1b09 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: e3d70fd4-fbdf-4501-b67e-8a95702b2851 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:55:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: dc37ed2f-0e43-48ba-9bde-c77a19d35975 kind: Event lastTimestamp: "2026-07-09T09:56:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:47Z" name: error-404-isvc-42ebc-predictor.18c096846381e4e3 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: 729b0872-60ee-4704-8791-cdba1088f587 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42ebc namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 2c4ebbbb-ccf1-43da-9261-af08d4c8dd45 kind: Event lastTimestamp: "2026-07-09T09:56:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:05Z" name: error-404-isvc-42ebc.18c0967d658a4ed7 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: 80e4fbeb-f07d-4832-8570-ae3825ccd172 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42ebc namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 2c4ebbbb-ccf1-43da-9261-af08d4c8dd45 kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-42ebc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-42ebc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:55:17Z" name: error-404-isvc-42ebc.18c0967d6608228b namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 8677240c-0c4d-4f07-81d8-290e3a59b69d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42ebc namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 2c4ebbbb-ccf1-43da-9261-af08d4c8dd45 kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-42ebc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:55:17Z" name: error-404-isvc-42ebc.18c0967d660875f4 namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: 10772bd2-c171-41aa-833c-bc1db6f1834a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42ebc namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: 2c4ebbbb-ccf1-43da-9261-af08d4c8dd45 kind: Event lastTimestamp: "2026-07-09T09:56:05Z" message: InferenceService [error-404-isvc-42ebc] is Ready metadata: creationTimestamp: "2026-07-09T09:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:05Z" name: error-404-isvc-42ebc.18c0968880a0667e namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 3c3c5855-b7dd-4807-997c-e241fbc13765 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T10:05:28.678928Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26083" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T10:05:28Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c0970ba34de7cd namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: b1616087-0cc2-4f25-84c8-5e992b7e8650 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26093" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T10:05:29Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c0970bbd268656 namespace: kserve-ci-e2e-test resourceVersion: "26111" uid: 895948ae-e6b6-43fb-8584-65f0ac50288b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c0970bbec10021 namespace: kserve-ci-e2e-test resourceVersion: "26114" uid: 0fe26604-568a-4244-b4b5-ebe19c811725 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c0970bc1bb0b1b namespace: kserve-ci-e2e-test resourceVersion: "26116" uid: bdf62c22-5793-46cd-847f-1e7355053d62 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c0970bc2255269 namespace: kserve-ci-e2e-test resourceVersion: "26117" uid: eecc4111-0139-48fc-b31e-fa46f27157dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c0970bc22e2eba namespace: kserve-ci-e2e-test resourceVersion: "26118" uid: 3e9e4665-3a2c-45d1-a77b-3e6e9051b655 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c0970bc4c98304 namespace: kserve-ci-e2e-test resourceVersion: "26119" uid: 723a983f-911c-4bd1-b307-e0edb035b29f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c0970bc553dffc namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: be97b4ec-be89-46be-9d44-b0725c902d13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:05:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:06:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:07Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c0970c35444539 namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: 8a59cd8e-61bd-4a37-a483-961247a44e2e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:22:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:22:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:22:48Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c097fda7a27564 namespace: kserve-ci-e2e-test resourceVersion: "33796" uid: 15a33759-242e-43cf-b51a-a91a72b45d1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:22:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 87a70fe9-fdf0-43ef-a105-083311069702 kind: Event lastTimestamp: "2026-07-09T10:22:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:22:48Z" name: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg.18c097fda7a2f565 namespace: kserve-ci-e2e-test resourceVersion: "33797" uid: a29a6c56-f7fa-420e-96f8-8b0503682faa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-52f9b-predictor-78498cf5bc namespace: kserve-ci-e2e-test resourceVersion: "26075" uid: 118c91c0-b0f3-4d2d-b475-f7c7f49a618e kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: 'Created pod: error-404-isvc-52f9b-predictor-78498cf5bc-mcbjg' metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:28Z" name: error-404-isvc-52f9b-predictor-78498cf5bc.18c0970ba2ed4111 namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: c0ccca44-8196-4dc4-9c71-1e642f7bfa0b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: c562c25e-fa8f-4592-95a9-5c4b4e4842b3 kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: Scaled up replica set error-404-isvc-52f9b-predictor-78498cf5bc from 0 to 1 metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:28Z" name: error-404-isvc-52f9b-predictor.18c0970ba121f038 namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 5ed43fb7-c0f4-4863-8503-855df7912bf1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: 28c7ee53-a7ea-4d0b-9c48-7c4b546a2fed kind: Event lastTimestamp: "2026-07-09T10:05:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:58Z" name: error-404-isvc-52f9b-predictor.18c0970f1f9cbde8 namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: a4b49beb-78f9-47fb-ae12-0afd31d935a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: 28c7ee53-a7ea-4d0b-9c48-7c4b546a2fed kind: Event lastTimestamp: "2026-07-09T10:05:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:58Z" name: error-404-isvc-52f9b-predictor.18c0970f20022bae namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: 62a12810-dedc-4817-98de-f9df5dd89d17 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T10:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: 28c7ee53-a7ea-4d0b-9c48-7c4b546a2fed kind: Event lastTimestamp: "2026-07-09T10:06:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:43Z" name: error-404-isvc-52f9b-predictor.18c097161d59ed29 namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: 91b7326b-6082-48d7-add8-bf7212f74edb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T10:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: 28c7ee53-a7ea-4d0b-9c48-7c4b546a2fed kind: Event lastTimestamp: "2026-07-09T10:06:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:43Z" name: error-404-isvc-52f9b-predictor.18c097161dc253a9 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: 50af223b-b6a3-481e-93ee-da771da0503f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26061" uid: db643fc2-f573-4b87-becb-8ed1e1ca7aac kind: Event lastTimestamp: "2026-07-09T10:06:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:17Z" name: error-404-isvc-52f9b.18c0970ba152e0f5 namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: be81be0c-8803-43a0-8533-0b589a9bd673 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26061" uid: db643fc2-f573-4b87-becb-8ed1e1ca7aac kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-52f9b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-52f9b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:28Z" name: error-404-isvc-52f9b.18c0970ba1a94252 namespace: kserve-ci-e2e-test resourceVersion: "26080" uid: 392a337d-538a-48f2-9408-70c68c933c33 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26061" uid: db643fc2-f573-4b87-becb-8ed1e1ca7aac kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-52f9b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:28Z" name: error-404-isvc-52f9b.18c0970ba1a97ec0 namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: 2b5df183-41b8-45e2-9cd9-67c4328239e9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: db643fc2-f573-4b87-becb-8ed1e1ca7aac kind: Event lastTimestamp: "2026-07-09T10:06:17Z" message: InferenceService [error-404-isvc-52f9b] is Ready metadata: creationTimestamp: "2026-07-09T10:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:17Z" name: error-404-isvc-52f9b.18c09716f1aacc74 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: fb716904-eebf-4a4f-8a91-3fb8a022b0fa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:45:48.623086Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8b307-predictor-59c78cddf5-l7zxp to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:45:48Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c095f8e27ea76f namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 92d6a025-879a-44a4-90d7-bf307eeb7812 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:45:49Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c095f8fbb2ac4e namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: f19b9b82-525c-45e0-a208-b37751e5ff78 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c095f8fd168ee7 namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 84b4fb93-52dc-4020-8c7d-5dbfae4a8943 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c095f90016950d namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: e22fa3d2-e6e0-4d30-82a6-f623f65f207d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c095f900849aaf namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 0945c115-4b31-468a-9efc-4b340ce81760 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c095f9008d319c namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: e04c713e-fd5c-4a5d-b6ff-6e88a3d4a610 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c095f903438c4f namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 56252903-b29e-4387-9d47-46153d2c31d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c095f903ae1b2a namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: 39aa9849-66ff-4a6f-afb9-4889700d01c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:46:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:26Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c095f9648a312e namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: 2e772a17-d360-4272-8b26-18b16852871c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c0967a0796ecc2 namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: f1a3c348-6049-4d0b-b495-d5dcdef41c08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 0c5d66b0-d347-4b61-bbf0-e000397703be kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp.18c0967a07979454 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 223611b1-8000-4297-878a-ee5e22e3d15d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8b307-predictor-59c78cddf5 namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: f6ac255e-f211-4be4-ba8a-7d67d08914f3 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: 'Created pod: error-404-isvc-8b307-predictor-59c78cddf5-l7zxp' metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:48Z" name: error-404-isvc-8b307-predictor-59c78cddf5.18c095f8e2106b0f namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 805a12a0-d8fa-46ff-a518-0cd3702d4536 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 58e1c6ef-e718-426b-978e-a0c92f1d0cc3 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Scaled up replica set error-404-isvc-8b307-predictor-59c78cddf5 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:48Z" name: error-404-isvc-8b307-predictor.18c095f8e0574272 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 3493ea04-ab54-4805-9c72-f8ab1961e4fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:46:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 66fd1e39-b0e5-4935-ba6a-955f83500730 kind: Event lastTimestamp: "2026-07-09T09:46:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:18Z" name: error-404-isvc-8b307-predictor.18c095fc5ee86a84 namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: fe22e0b2-027e-4776-8a5f-6c56f64d81d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:46:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 66fd1e39-b0e5-4935-ba6a-955f83500730 kind: Event lastTimestamp: "2026-07-09T09:46:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:18Z" name: error-404-isvc-8b307-predictor.18c095fc5f4cbd0b namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: e619d3d5-9de1-4449-a57d-d241033e1aed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 66fd1e39-b0e5-4935-ba6a-955f83500730 kind: Event lastTimestamp: "2026-07-09T09:47:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:18Z" name: error-404-isvc-8b307-predictor.18c096035cd5e6a2 namespace: kserve-ci-e2e-test resourceVersion: "17081" uid: 74e01e39-2032-4e6b-b0d2-55151b83dc86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:46:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 66fd1e39-b0e5-4935-ba6a-955f83500730 kind: Event lastTimestamp: "2026-07-09T09:47:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:18Z" name: error-404-isvc-8b307-predictor.18c096035d519957 namespace: kserve-ci-e2e-test resourceVersion: "17082" uid: 2578fcec-f3e5-46e1-a71b-7ba0a5f30f7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 4c9c80ea-abae-4d33-886a-3331c3a33e06 kind: Event lastTimestamp: "2026-07-09T09:46:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:46:36Z" name: error-404-isvc-8b307.18c095f8e06ed96c namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: e61eb1db-eba0-4e1f-b8bc-232fb5f56735 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 4c9c80ea-abae-4d33-886a-3331c3a33e06 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: 'Failed to update status for InferenceService "error-404-isvc-8b307": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8b307": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:48Z" name: error-404-isvc-8b307.18c095f8e101abe5 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: bafb7ef8-22ae-4537-882b-65fde9d4ea7f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 4c9c80ea-abae-4d33-886a-3331c3a33e06 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8b307": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:48Z" name: error-404-isvc-8b307.18c095f8e101da40 namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 00fb0cf6-75f8-4c94-aa82-d487d659153c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16722" uid: 4c9c80ea-abae-4d33-886a-3331c3a33e06 kind: Event lastTimestamp: "2026-07-09T09:46:36Z" message: InferenceService [error-404-isvc-8b307] is Ready metadata: creationTimestamp: "2026-07-09T09:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:46:36Z" name: error-404-isvc-8b307.18c096042004be2c namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 1341c4fa-3cbf-4e02-89e5-fa2cac1bfa23 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:44:52.646403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:44:52Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c095ebda064405 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 84ef5832-f182-4cd7-a196-c744d8027f5c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:44:53Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c095ebf3a82310 namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 428c7c57-8f0d-468f-9fa2-ba234fe3ec7b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c095ebf505eac9 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: b573cf40-42d0-4c48-895b-325c7eb143c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c095ebf78df21a namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: c55cfa02-ae0c-4ff9-8602-ddb8bb857ef1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c095ebf7ef49bd namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: b60df8c4-fedb-4385-8cba-af9d88e143ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c095ebf7f7bab3 namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 349c3566-1ab2-4024-be2b-dce3eebef729 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c095ebfa685b05 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: 823fd6b3-db3a-40e9-abbe-989f811cbc6f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c095ebfae19dcd namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 4d9b502c-56e8-4133-a339-dca03b9ecbcc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:44:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:45:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:30Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c095ec4bbf555b namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 5c84fd9e-4236-4779-8c00-6b224bb59fe4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c0966d07b542c8 namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: dca72616-6392-43b8-854a-d989300cf2c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: e74da4a0-f711-45a7-b0dc-db566880a5c2 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6.18c0966d07bbeff0 namespace: kserve-ci-e2e-test resourceVersion: "19955" uid: 2fb44941-8571-475e-a2f6-fa925d043e13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8d1ee-predictor-5565fb77bf namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 186880c1-9d61-487e-9db2-c7914e99922e kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: 'Created pod: error-404-isvc-8d1ee-predictor-5565fb77bf-qzfp6' metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:52Z" name: error-404-isvc-8d1ee-predictor-5565fb77bf.18c095ebd976ed2f namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 59c67a52-0a7a-4dec-8886-a6b4c09c4578 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15533" uid: c55e60ff-4d0b-4061-a985-5f350fe764f8 kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: Scaled up replica set error-404-isvc-8d1ee-predictor-5565fb77bf from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:52Z" name: error-404-isvc-8d1ee-predictor.18c095ebd7bd2b8f namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 32f9ae13-6583-4d32-b7e6-b6007865cae0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:45:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: c1a65314-0d9b-4e0b-9d52-41eac3023203 kind: Event lastTimestamp: "2026-07-09T09:45:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:22Z" name: error-404-isvc-8d1ee-predictor.18c095ef5634d7de namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: 7f2a8566-4090-449a-b99c-ef1281163b94 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:45:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: c1a65314-0d9b-4e0b-9d52-41eac3023203 kind: Event lastTimestamp: "2026-07-09T09:45:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:22Z" name: error-404-isvc-8d1ee-predictor.18c095ef569f8384 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 4c1b4e46-caa7-47d7-9e41-470dab2af526 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:45:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: c1a65314-0d9b-4e0b-9d52-41eac3023203 kind: Event lastTimestamp: "2026-07-09T09:46:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:07Z" name: error-404-isvc-8d1ee-predictor.18c095f654c105c2 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: cd90fa19-ae0b-4045-84bf-4e2f45f9dbbd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:45:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: c1a65314-0d9b-4e0b-9d52-41eac3023203 kind: Event lastTimestamp: "2026-07-09T09:46:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:07Z" name: error-404-isvc-8d1ee-predictor.18c095f6550cadaa namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 6403ea23-9f9a-4b4d-90d7-9fc38d775aee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 253e3716-aa4e-4d36-9eb6-0dc8a0c73981 kind: Event lastTimestamp: "2026-07-09T09:45:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:40Z" name: error-404-isvc-8d1ee.18c095ebd7b3c87c namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: f30be944-beaf-475a-9f59-7fab81c66526 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 253e3716-aa4e-4d36-9eb6-0dc8a0c73981 kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-8d1ee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:52Z" name: error-404-isvc-8d1ee.18c095ebd85a7024 namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 66547c1b-8ff5-44d3-be11-5acff37dda75 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 253e3716-aa4e-4d36-9eb6-0dc8a0c73981 kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:52Z" name: error-404-isvc-8d1ee.18c095ebd85abb0d namespace: kserve-ci-e2e-test resourceVersion: "15557" uid: 0ca9d0a1-18cd-4673-b684-98e5e29d2909 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 253e3716-aa4e-4d36-9eb6-0dc8a0c73981 kind: Event lastTimestamp: "2026-07-09T09:45:40Z" message: InferenceService [error-404-isvc-8d1ee] is Ready metadata: creationTimestamp: "2026-07-09T09:45:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:40Z" name: error-404-isvc-8d1ee.18c095f70b271d42 namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 8d4155ec-9a61-4394-a493-d821a5520d95 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T10:04:32.654748Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25262" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9d678-predictor-6b97f5f755-bbpfq to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T10:04:32Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096fe9800af73 namespace: kserve-ci-e2e-test resourceVersion: "25266" uid: 120d0952-af20-4dde-9c00-0db47c1cee60 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:04:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9d678-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:33Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096fea3f63609 namespace: kserve-ci-e2e-test resourceVersion: "25322" uid: 9355cfde-41da-4d43-ad8f-7b52c1b6f4f1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:04:34Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T10:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T10:04:34Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096ff0ade2fca namespace: kserve-ci-e2e-test resourceVersion: "25361" uid: 5969f137-0068-46f8-aa4a-59ffbf434b8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:04:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T10:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:34Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096ff0c2e84d6 namespace: kserve-ci-e2e-test resourceVersion: "25363" uid: a784e97e-32e3-438a-bd86-1385e46ff58a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:04:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T10:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:34Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096ff0f852ed1 namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: b84654f2-57e6-4140-8ef9-2c83a6d537e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:04:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T10:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:34Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096ff0ff3a899 namespace: kserve-ci-e2e-test resourceVersion: "25367" uid: c9a18dd8-bd0e-48b3-85ba-daed6fe67fdb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:04:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T10:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:34Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096ff0ffd01f3 namespace: kserve-ci-e2e-test resourceVersion: "25368" uid: 97d775c9-f005-449e-9ca0-353532513eb9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:04:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T10:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:34Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096ff13669186 namespace: kserve-ci-e2e-test resourceVersion: "25370" uid: aa478c4e-0f57-47f4-878c-f883044fa0f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:04:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:34Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096ff13dd4c0d namespace: kserve-ci-e2e-test resourceVersion: "25371" uid: 100911b7-1d78-4dac-b32c-4e1337289429 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:04:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:05:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:11Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c096ff58103e5d namespace: kserve-ci-e2e-test resourceVersion: "25847" uid: c9934e98-7d03-4912-8bc6-09d7ece64a05 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:42Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c0970ef6487457 namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: f30916a2-2c37-4615-8bf1-4b7bf0b48598 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 15e4fa1a-11e6-4f3a-8571-12fe9ea6f9ed kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:42Z" name: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq.18c0970ef64a359e namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: 8c373842-1257-4487-bcf1-f9a304147f93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9d678-predictor-6b97f5f755 namespace: kserve-ci-e2e-test resourceVersion: "25247" uid: 4f763f01-1dac-4a63-87a9-7ec0c5b2ef25 kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: 'Created pod: error-404-isvc-9d678-predictor-6b97f5f755-bbpfq' metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:04:32Z" name: error-404-isvc-9d678-predictor-6b97f5f755.18c096fe979ad941 namespace: kserve-ci-e2e-test resourceVersion: "25263" uid: b4a53dc2-bac0-43a5-a4f5-3d3297ab2909 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 384f203c-eda9-44e7-b11d-75da8fd0ad23 kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: Scaled up replica set error-404-isvc-9d678-predictor-6b97f5f755 from 0 to 1 metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:04:32Z" name: error-404-isvc-9d678-predictor.18c096fe95a2287b namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: 16fc684f-cc9d-42a9-b440-df38bd2806f4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:04:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25246" uid: b6f22073-afdf-41d0-870a-238f06e6ecfb kind: Event lastTimestamp: "2026-07-09T10:05:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:04:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:02Z" name: error-404-isvc-9d678-predictor.18c09702142427be namespace: kserve-ci-e2e-test resourceVersion: "25779" uid: 24c5fc46-4b7e-4502-887c-b34bdc5fdc03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:04:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25246" uid: b6f22073-afdf-41d0-870a-238f06e6ecfb kind: Event lastTimestamp: "2026-07-09T10:05:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:04:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:02Z" name: error-404-isvc-9d678-predictor.18c09702148a968e namespace: kserve-ci-e2e-test resourceVersion: "25780" uid: 5eb0b646-3c7f-45d2-bf58-67a820d44160 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25669" uid: b6f22073-afdf-41d0-870a-238f06e6ecfb kind: Event lastTimestamp: "2026-07-09T10:05:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:32Z" name: error-404-isvc-9d678-predictor.18c0970911ed78fe namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: 801406b4-ad01-4d61-9832-fcef76eaafd6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25669" uid: b6f22073-afdf-41d0-870a-238f06e6ecfb kind: Event lastTimestamp: "2026-07-09T10:05:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:32Z" name: error-404-isvc-9d678-predictor.18c0970912572fd3 namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: 23cf3eb4-d6ce-4fc7-abe2-058b60d8a316 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d678 namespace: kserve-ci-e2e-test resourceVersion: "25220" uid: f8efde5f-9f98-48a0-95d0-b32be4f5960c kind: Event lastTimestamp: "2026-07-09T10:05:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:22Z" name: error-404-isvc-9d678.18c096fe95ab1cd3 namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: e8df7c51-2b95-4067-92fb-1be3c92228d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d678 namespace: kserve-ci-e2e-test resourceVersion: "25220" uid: f8efde5f-9f98-48a0-95d0-b32be4f5960c kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: 'Failed to update status for InferenceService "error-404-isvc-9d678": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9d678": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:04:32Z" name: error-404-isvc-9d678.18c096fe96331f39 namespace: kserve-ci-e2e-test resourceVersion: "25256" uid: 18498a55-7e5d-4bd8-bf4c-05ae13868c17 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d678 namespace: kserve-ci-e2e-test resourceVersion: "25220" uid: f8efde5f-9f98-48a0-95d0-b32be4f5960c kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9d678": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:04:32Z" name: error-404-isvc-9d678.18c096fe96336fd7 namespace: kserve-ci-e2e-test resourceVersion: "25258" uid: 3a4d5874-fa01-44a4-993c-7aa32e062973 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9d678 namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: f8efde5f-9f98-48a0-95d0-b32be4f5960c kind: Event lastTimestamp: "2026-07-09T10:05:21Z" message: InferenceService [error-404-isvc-9d678] is Ready metadata: creationTimestamp: "2026-07-09T10:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:21Z" name: error-404-isvc-9d678.18c0970a138e5761 namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 21615d6d-0cdc-4932-8d43-3ffa7d419b31 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:55:03.418023Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0967a0ed99183 namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 30fbbae1-f05b-481e-b9a9-d631b6dd4f44 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0967a28e6ccf2 namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: f8b7dc32-b346-4b86-84bc-b7f21732c8fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0967a2a611b39 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: 408e96c2-4c17-4849-9faa-42ec7d96a6e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0967a2d15216f namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: 163ae849-e4ea-47e4-abfc-f0eb7669eae7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0967a2d83a8f6 namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: 807858ce-8601-4f13-bc2d-89e16dec1fe1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0967a2d8deb5a namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 1e31741c-98a4-4357-a26d-02c05685b8da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0967a30524c71 namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 2af6ff7f-f627-4146-ace0-395a3ed87e4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0967a30c3227d namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 5d87cc91-151a-4469-8855-05ef151d4165 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:55:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:41Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0967a7e042253 namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: fac6fbc7-3de5-43c4-a4e9-f4ab76be9901 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:13Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0968a5f39b892 namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: 70c54cea-7ba5-4fad-b682-dd449174b074 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:13Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0968a5f3a39e5 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 8eafc51e-5e79-429c-9af4-a34918be2d3f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: dd8634d0-e555-4d48-bccd-5960c29566a6 kind: Event lastTimestamp: "2026-07-09T09:56:16Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:16Z" name: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl.18c0968b064f999d namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 494e7693-d013-46cf-8ee8-455c1daa03f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a3d13-predictor-58658c6cf9 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: b443ef59-553a-498c-94cf-ed3b6f897063 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: 'Created pod: error-404-isvc-a3d13-predictor-58658c6cf9-pr7kl' metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor-58658c6cf9.18c0967a0e4bfea4 namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: 45d779a2-7112-458c-a320-40e65b206c59 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 9b037e0d-4ad2-4465-9e85-003422d668bc kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Scaled up replica set error-404-isvc-a3d13-predictor-58658c6cf9 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13-predictor.18c0967a0c90f117 namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: 4a91b81d-d989-4df8-a214-68f4514d8dd3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: c6a3a67f-3229-46db-b62c-f0d845feb0b5 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:18Z" name: error-404-isvc-a3d13-predictor.18c0967d8af9facd namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 49b217ce-1b6f-4586-bee2-949bb5f9d9f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: c6a3a67f-3229-46db-b62c-f0d845feb0b5 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:18Z" name: error-404-isvc-a3d13-predictor.18c0967d8b628097 namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: 07ac8aa2-c14f-4825-afc1-185a46134fb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:55:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: c6a3a67f-3229-46db-b62c-f0d845feb0b5 kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:03Z" name: error-404-isvc-a3d13-predictor.18c096810a1544b8 namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: b7baa4ee-ec3c-4461-8015-faf6724628a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:55:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: c6a3a67f-3229-46db-b62c-f0d845feb0b5 kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:03Z" name: error-404-isvc-a3d13-predictor.18c096810aa24092 namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: f090b272-f878-48db-a622-2efe89b731e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3d13 namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: 698a92df-c43d-4f20-a1e7-35c651fda5b4 kind: Event lastTimestamp: "2026-07-09T09:55:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:55:51Z" name: error-404-isvc-a3d13.18c0967a0c99c16e namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: c2de2e5b-d2f5-444b-964c-523f49695832 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3d13 namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: 698a92df-c43d-4f20-a1e7-35c651fda5b4 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-a3d13": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a3d13": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13.18c0967a0cf6a8e6 namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: f36e5e3c-3677-4423-845e-c3a95c5970fc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3d13 namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: 698a92df-c43d-4f20-a1e7-35c651fda5b4 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a3d13": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:55:03Z" name: error-404-isvc-a3d13.18c0967a0cf6e5ed namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 4d9d6e7d-0429-4bff-8997-8198ceaf1ace reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3d13 namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 698a92df-c43d-4f20-a1e7-35c651fda5b4 kind: Event lastTimestamp: "2026-07-09T09:55:51Z" message: InferenceService [error-404-isvc-a3d13] is Ready metadata: creationTimestamp: "2026-07-09T09:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:55:51Z" name: error-404-isvc-a3d13.18c0968539283c53 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 6ad104d2-75ff-4b53-8bea-7f8bc38bf357 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T10:05:42.985057Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a7fc2-predictor-78f845d769-bmbbs to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T10:05:42Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970ef803f8d8 namespace: kserve-ci-e2e-test resourceVersion: "26383" uid: f1879090-7fee-4a8e-9a14-868376ef72cf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:05:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a7fc2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:43Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970f07198603 namespace: kserve-ci-e2e-test resourceVersion: "26400" uid: 8e485b3e-a7f7-4590-963c-bd2fcc3003a0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:05:44Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T10:05:44Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970f3554ecfa namespace: kserve-ci-e2e-test resourceVersion: "26421" uid: 346fb47e-ecd8-4c09-80b2-4831b6dba57e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:05:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:44Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970f36da9327 namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 22169b21-e658-4bac-9d15-89ad76d5c955 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:05:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:44Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970f39d826a1 namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 4f9e5d3e-1299-4f30-ab7c-26c166003b3e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:05:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:44Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970f3a4148e5 namespace: kserve-ci-e2e-test resourceVersion: "26428" uid: 798109b2-def1-4ddb-ba66-a4185d04c21c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:05:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:44Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970f3a4a203d namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: 1e3a77c1-fd04-4508-ad56-0c235de257a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:05:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:44Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970f3d255a8f namespace: kserve-ci-e2e-test resourceVersion: "26430" uid: 93491bc8-bf48-48dd-abb5-895ecdb87721 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:05:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:44Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970f3d95ffbe namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: beae1bdd-3f5d-4c7d-bd64-bd339d29da34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:05:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:06:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:05:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:21Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c0970f7bfc2998 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 6d9b6c37-0593-4282-957a-ec201031d397 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:14:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:14:57Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c097902fb6fd80 namespace: kserve-ci-e2e-test resourceVersion: "30536" uid: 6091b6a1-72a8-49ad-9fb2-f832e9a4d73e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 3a6481a6-4e72-4e32-adb4-d4a9932d530c kind: Event lastTimestamp: "2026-07-09T10:14:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:14:57Z" name: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs.18c097902fb78fc9 namespace: kserve-ci-e2e-test resourceVersion: "30537" uid: a442211a-6544-40e3-81fa-0a95571b1d6f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a7fc2-predictor-78f845d769 namespace: kserve-ci-e2e-test resourceVersion: "26363" uid: 4c1c7b9d-ad2e-44aa-a3f1-4765e46467ae kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: 'Created pod: error-404-isvc-a7fc2-predictor-78f845d769-bmbbs' metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:42Z" name: error-404-isvc-a7fc2-predictor-78f845d769.18c0970ef76ffbc0 namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: e681d51d-0525-4380-9c91-da280bd3d456 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26357" uid: 2f10d139-203c-44f9-93cb-92141605fdb6 kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: Scaled up replica set error-404-isvc-a7fc2-predictor-78f845d769 from 0 to 1 metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:42Z" name: error-404-isvc-a7fc2-predictor.18c0970ef53edd8b namespace: kserve-ci-e2e-test resourceVersion: "26367" uid: 1202c673-842d-4458-af79-eb7af6e7f96e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: dd3218c4-b697-4efe-b2fb-668cb5e261c2 kind: Event lastTimestamp: "2026-07-09T10:06:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:12Z" name: error-404-isvc-a7fc2-predictor.18c0971273ce1a50 namespace: kserve-ci-e2e-test resourceVersion: "26679" uid: 11673695-22e1-4ca6-a589-e1b31f4147a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: dd3218c4-b697-4efe-b2fb-668cb5e261c2 kind: Event lastTimestamp: "2026-07-09T10:06:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:12Z" name: error-404-isvc-a7fc2-predictor.18c09712741342f0 namespace: kserve-ci-e2e-test resourceVersion: "26680" uid: 7094a393-e7d1-4ca3-8a39-c26da3f90582 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T10:06:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: dd3218c4-b697-4efe-b2fb-668cb5e261c2 kind: Event lastTimestamp: "2026-07-09T10:06:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:58Z" name: error-404-isvc-a7fc2-predictor.18c09719718d3aad namespace: kserve-ci-e2e-test resourceVersion: "27143" uid: 57ade31f-5c37-40c4-b971-93e9abe8cff8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T10:06:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: dd3218c4-b697-4efe-b2fb-668cb5e261c2 kind: Event lastTimestamp: "2026-07-09T10:06:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:58Z" name: error-404-isvc-a7fc2-predictor.18c0971971f34c35 namespace: kserve-ci-e2e-test resourceVersion: "27144" uid: e9ed7dff-3108-4058-b60a-cd20c81a8e97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26341" uid: 5b270a0d-1702-486d-8b1e-a3f580b7115c kind: Event lastTimestamp: "2026-07-09T10:06:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:31Z" name: error-404-isvc-a7fc2.18c0970ef58f0c21 namespace: kserve-ci-e2e-test resourceVersion: "26899" uid: 84db6cb7-80d4-49b8-ad85-4473a4079306 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26341" uid: 5b270a0d-1702-486d-8b1e-a3f580b7115c kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: 'Failed to update status for InferenceService "error-404-isvc-a7fc2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a7fc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:42Z" name: error-404-isvc-a7fc2.18c0970ef620c899 namespace: kserve-ci-e2e-test resourceVersion: "26372" uid: 81006566-d4f5-4c59-b3dd-1def3dcc50e2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26341" uid: 5b270a0d-1702-486d-8b1e-a3f580b7115c kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a7fc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:42Z" name: error-404-isvc-a7fc2.18c0970ef62120fc namespace: kserve-ci-e2e-test resourceVersion: "26375" uid: 68143174-7a95-4d3f-a414-9a70d5025359 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: 5b270a0d-1702-486d-8b1e-a3f580b7115c kind: Event lastTimestamp: "2026-07-09T10:06:31Z" message: InferenceService [error-404-isvc-a7fc2] is Ready metadata: creationTimestamp: "2026-07-09T10:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:31Z" name: error-404-isvc-a7fc2.18c0971a37674668 namespace: kserve-ci-e2e-test resourceVersion: "26898" uid: ed4b35f7-e1ca-4cdf-909d-bcb1c3570cc2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:54:07.489198Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19956" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b04db-predictor-f486cdf95-s8fmc to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0966d093b7008 namespace: kserve-ci-e2e-test resourceVersion: "19961" uid: 4d86b89c-a371-4d56-92da-4d8eadabf0f9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19965" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0966d22d8b5f1 namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: 29d6ceb7-3ede-473d-9811-301f4c312cb1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0966d2437dcdb namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: b58d4670-af7a-4952-8bb6-7f77727508ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0966d274e82ce namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: 9ae88cc1-f198-4d49-857f-42dc43d832e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:54:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:08Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0966d27c1a57a namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: 628e1270-cf32-420b-8a07-4b6ed70911c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:54:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:08Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0966d27cafe45 namespace: kserve-ci-e2e-test resourceVersion: "19984" uid: 1660703e-d8cb-42a1-9a51-81d86f95f492 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:54:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:08Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0966d2a7443da namespace: kserve-ci-e2e-test resourceVersion: "19985" uid: a91f890c-f287-45ee-87a5-c0e109a4e2f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:54:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:08Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0966d2ae903f5 namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: f530e553-210e-472e-9373-3ac47739695f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:54:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:54:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:54:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:46Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0966da0720c5b namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: db5719b3-4cc2-482d-81ac-debf9786e4b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:17Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0967d61c9b9f4 namespace: kserve-ci-e2e-test resourceVersion: "20807" uid: d9b04ab4-adb7-496c-9b18-45d1b387f94a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:17Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0967d61ca360a namespace: kserve-ci-e2e-test resourceVersion: "20810" uid: da7b4c3e-2195-4d0a-b4b6-c6c046dee79c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 1145536a-0c40-4de2-8755-d80cda6b21ba kind: Event lastTimestamp: "2026-07-09T09:55:22Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-09T09:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:22Z" name: error-404-isvc-b04db-predictor-f486cdf95-s8fmc.18c0967e645ab340 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: dc2dddc1-9abe-499c-b6c7-14239bee0e85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b04db-predictor-f486cdf95 namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 13fce8ae-15cb-4bbc-91e6-8482804f1fe8 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: 'Created pod: error-404-isvc-b04db-predictor-f486cdf95-s8fmc' metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-b04db-predictor-f486cdf95.18c0966d08b23fa4 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 3254697d-4028-4170-bbc2-26a7640ed330 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "19934" uid: e97daaf5-c758-4772-bcbf-85e533f3d37b kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: Scaled up replica set error-404-isvc-b04db-predictor-f486cdf95 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-b04db-predictor.18c0966d06820e84 namespace: kserve-ci-e2e-test resourceVersion: "19947" uid: 7e9b30f9-5d18-44a0-8a30-ba7e9ccdb89f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: 412494a6-9ab2-4eb1-8661-eafe0f9337ae kind: Event lastTimestamp: "2026-07-09T09:54:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:22Z" name: error-404-isvc-b04db-predictor.18c09670852ccdb1 namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: b03eed54-8935-4353-bbf5-db41d4c7431c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: 412494a6-9ab2-4eb1-8661-eafe0f9337ae kind: Event lastTimestamp: "2026-07-09T09:54:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:22Z" name: error-404-isvc-b04db-predictor.18c09670857824fb namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: 538486bd-8029-49ec-9e1e-8a3a56603514 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:54:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 412494a6-9ab2-4eb1-8661-eafe0f9337ae kind: Event lastTimestamp: "2026-07-09T09:55:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:54:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:07Z" name: error-404-isvc-b04db-predictor.18c09674042309d1 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 026efc5e-a062-4032-b6ac-057412ba022b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:54:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 412494a6-9ab2-4eb1-8661-eafe0f9337ae kind: Event lastTimestamp: "2026-07-09T09:55:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:54:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:07Z" name: error-404-isvc-b04db-predictor.18c0967404871112 namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 4abeabda-7ce6-4a9f-9450-10adf68fc786 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b04db namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: 49a73414-15a6-49d5-87fc-03b108913568 kind: Event lastTimestamp: "2026-07-09T09:54:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:56Z" name: error-404-isvc-b04db.18c0966d06f79792 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: b7433bbb-1032-4cb3-be11-06b90bc6e0f6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b04db namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: 49a73414-15a6-49d5-87fc-03b108913568 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: 'Failed to update status for InferenceService "error-404-isvc-b04db": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b04db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-b04db.18c0966d079903c0 namespace: kserve-ci-e2e-test resourceVersion: "19951" uid: 5bd8afed-1cd5-4b4a-b9af-b8d494fe36f8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b04db namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: 49a73414-15a6-49d5-87fc-03b108913568 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b04db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:07Z" name: error-404-isvc-b04db.18c0966d0799666d namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: 71777e55-3428-4e10-ad5f-8633c3c3f4a8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b04db namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 49a73414-15a6-49d5-87fc-03b108913568 kind: Event lastTimestamp: "2026-07-09T09:54:56Z" message: InferenceService [error-404-isvc-b04db] is Ready metadata: creationTimestamp: "2026-07-09T09:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:56Z" name: error-404-isvc-b04db.18c096785bbcf4f6 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: db74063d-0bc3-481f-a565-50a8dcfc55c1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:43:18.356980Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14401" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 to ip-10-0-135-204.ec2.internal metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:43:18Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095d5e5efda5d namespace: kserve-ci-e2e-test resourceVersion: "14406" uid: 5789895d-2e7a-4178-babc-64d56382c18c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:18Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095d5f0aea1a5 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: 3c6b4d44-11ed-42ab-b4ae-6e9b060a80be reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:43:19Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095d623273283 namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: ad71c2bc-5081-4842-886f-75cfed0b1f09 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095d6248f5a6f namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: 25e42c0e-d3c9-48f6-9dff-3bfe6b4aa7ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095d62807bed2 namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: d9af6468-3423-4913-b877-b7367540b69b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095d6287eecaa namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 6af9e75c-ab95-4012-8bd1-6d40c0779719 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" metadata: creationTimestamp: "2026-07-09T09:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:23Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095d72adc0501 namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: ccba0ee8-fb09-4747-bc4f-0da72e9edf34 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:51Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" in 27.885s (27.885s including waiting). Image size: 1557794731 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:51Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095dda8f9aaf0 namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: f04fd326-dc07-4e2b-b254-d2bb6e094580 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:51Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095ddae5aad93 namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: 478f8560-8722-4fd9-9015-f4b67affa54c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:51Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095ddb0a57505 namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: 8c7fdf8f-59f9-4994-b816-05d37493108c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:51Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T09:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:51Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095ddb0b1cae9 namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 49508c6a-225f-4a73-b053-eb86e1f05ee5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.993s (2.993s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:54Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095de6326a9b6 namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 0a6b4552-2a75-4c95-aae0-72c0e8cfbca0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:54Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095de67bd93dc namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 375222c0-853a-4d4e-8141-7913917061b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:43:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:54Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095de683acea0 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: ee3e4e71-0ba6-4345-aa94-d254b8fad0e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T09:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:45:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:51Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095de6f51f573 namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: b471f65a-dede-478a-aef7-241651768daa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:48Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095f8db191c89 namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 748cc610-da62-426e-a4dc-30c90f4b297e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:48Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095f8db19afcb namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: f2df226b-ace7-4dee-9088-550798633802 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bd442d1d-ea1c-47cd-9b75-7f6eb7f61cbc kind: Event lastTimestamp: "2026-07-09T09:45:51Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:51Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899.18c095f971ad4b8b namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 63117e42-70c1-4fab-8ef5-4412c87ef696 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7c964dbd44 namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 1197c2d8-0a54-4702-9737-fab64199c3c0 kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7c964dbd44-rn899' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: isvc-sklearn-graph-1-predictor-7c964dbd44.18c095d5e57091fc namespace: kserve-ci-e2e-test resourceVersion: "14403" uid: 00be49f6-febd-4e11-934d-56aa7885846a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 546a422c-be90-41df-9c9b-1b22160e0f07 kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7c964dbd44 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: isvc-sklearn-graph-1-predictor.18c095d5dd7734be namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: 7b6d24a9-7ded-47fe-8387-917e1252d568 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: 65d89a20-ec2b-40a5-8eb5-e4bbdd8171ff kind: Event lastTimestamp: "2026-07-09T09:44:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:33Z" name: isvc-sklearn-graph-1-predictor.18c095d96688fdc9 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 39606c93-0eb8-4d4e-945f-40576317e7a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: 65d89a20-ec2b-40a5-8eb5-e4bbdd8171ff kind: Event lastTimestamp: "2026-07-09T09:44:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:33Z" name: isvc-sklearn-graph-1-predictor.18c095d9679e4fa6 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 47cd465f-c855-44d2-8bbe-1c9f9f02ac80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 65d89a20-ec2b-40a5-8eb5-e4bbdd8171ff kind: Event lastTimestamp: "2026-07-09T09:45:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:33Z" name: isvc-sklearn-graph-1-predictor.18c095e3e570154b namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: 3284ff75-5b2a-41cb-938c-b417cf3cdde8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 65d89a20-ec2b-40a5-8eb5-e4bbdd8171ff kind: Event lastTimestamp: "2026-07-09T09:45:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:33Z" name: isvc-sklearn-graph-1-predictor.18c095e3e5e53d4d namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: 1c41af16-fe92-4c59-a9c7-bef838ee3a00 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: 5389cea3-a9f9-4819-8413-0c563ed82ef4 kind: Event lastTimestamp: "2026-07-09T09:45:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:11Z" name: isvc-sklearn-graph-1.18c095d5e36197ba namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 5dd74a1f-0c45-4faa-826b-a3b9b73da37d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: 5389cea3-a9f9-4819-8413-0c563ed82ef4 kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:43:18Z" name: isvc-sklearn-graph-1.18c095d5e3d5e6ef namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: f34c48f6-159d-4f99-bd71-5c7b70c9fd5f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: 5389cea3-a9f9-4819-8413-0c563ed82ef4 kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:43:18Z" name: isvc-sklearn-graph-1.18c095d5e3d6250f namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: 45f70ac0-b0cb-483f-9c4b-ebd0e0345fa2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 5389cea3-a9f9-4819-8413-0c563ed82ef4 kind: Event lastTimestamp: "2026-07-09T09:45:11Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-09T09:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:11Z" name: isvc-sklearn-graph-1.18c095f02709f45d namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: 9e1e0648-79b3-4093-aa8d-1337c349e57c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:43:18.768047Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-84b568d556-46fps to ip-10-0-135-204.ec2.internal metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:43:18Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095d5fe702c5b namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: f938d557-8987-4299-9567-0b9daf83b8b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:43:19Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095d6181947e7 namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: 42d6a9bb-9c3d-4923-9e91-61b9ca1dc6da reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095d619a504a5 namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 93a752ac-38e6-414e-91f7-4b595e37105e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095d61c8fa782 namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: ecb803da-5a3e-4eef-8375-75fad02a89f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095d61d08d1be namespace: kserve-ci-e2e-test resourceVersion: "14555" uid: e85f711d-239e-4790-9fbe-f61af007d1ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" metadata: creationTimestamp: "2026-07-09T09:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:23Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095d72af4f1c3 namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 35deb06d-22c1-4560-9383-8b5b2fadbb84 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:51Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1684" in 27.932s (27.932s including waiting). Image size: 1557794731 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:51Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095ddabe4f05d namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: bf516baa-3044-484c-8548-4ca9c81503c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:51Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095ddb13f7a76 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: d39a003c-b72c-4d60-b971-64555eff5518 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:51Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095ddb2557cd4 namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 8255da04-1b17-4d7f-8b88-627a951be092 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:51Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T09:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:51Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095ddb2631fa7 namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: 89e564e9-2806-4ae7-89b4-2fe4da8bb189 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.945s (2.945s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:54Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095de61f3db57 namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: 38203c27-fa0c-4195-8c0a-b8f011bbf8e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:54Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095de67c69204 namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: 42eb3226-f3fb-4850-ae0c-a281f9ae72cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:43:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:54Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095de6833b072 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: f4281b66-543c-4251-b584-f39a37bb841c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T09:43:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:45:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:52Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095deab2703ec namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 178ae222-ae79-4971-a7e7-f78bfaa526a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:48Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095f8d7e59c0a namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: 33c63634-c114-48e0-a66a-d3a863c525bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:48Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095f8d7e5f3ec namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 386c0e67-9777-4ab1-bac3-76aa64534468 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-84b568d556-46fps namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: c8d2c03e-cf8b-4bcc-b354-11e93e4b3bca kind: Event lastTimestamp: "2026-07-09T09:45:52Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:52Z" name: isvc-sklearn-graph-2-predictor-84b568d556-46fps.18c095f9ad7cb7ef namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 70d03ca2-e9ac-46d7-ac85-fcb7b116528c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-84b568d556 namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: b16b0e1b-d965-460a-8e97-05f2cd00fe3a kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-84b568d556-46fps' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: isvc-sklearn-graph-2-predictor-84b568d556.18c095d5fdf458f5 namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: 939b8b86-25e5-4235-b0cb-6c681df2e516 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: ec3230a9-8fd8-4c7c-bdfa-3e00451f4d83 kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-84b568d556 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: isvc-sklearn-graph-2-predictor.18c095d5fb43606d namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: bb194fdc-0100-4a61-882d-aa2cae223ddf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 81c0aa8f-e764-4faa-8bc6-b2e0c26c3289 kind: Event lastTimestamp: "2026-07-09T09:44:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:33Z" name: isvc-sklearn-graph-2-predictor.18c095d979d11892 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: fc81de66-d4b5-4181-896d-4586bd011aa7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 81c0aa8f-e764-4faa-8bc6-b2e0c26c3289 kind: Event lastTimestamp: "2026-07-09T09:44:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:33Z" name: isvc-sklearn-graph-2-predictor.18c095d97a40237c namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 326a0cd7-cb2e-474e-9aa1-e207996c4a16 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:44:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: 81c0aa8f-e764-4faa-8bc6-b2e0c26c3289 kind: Event lastTimestamp: "2026-07-09T09:45:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:33Z" name: isvc-sklearn-graph-2-predictor.18c095eaf40beee8 namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: 3f208219-464c-40eb-a149-cfc442012861 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:44:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: 81c0aa8f-e764-4faa-8bc6-b2e0c26c3289 kind: Event lastTimestamp: "2026-07-09T09:45:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:33Z" name: isvc-sklearn-graph-2-predictor.18c095eaf49aed4e namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: 75474b3b-2724-4e60-8e7a-7e9b5f72a123 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: 1f71a387-f1bd-44a2-aaca-93dabd5284c9 kind: Event lastTimestamp: "2026-07-09T09:45:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:12Z" name: isvc-sklearn-graph-2.18c095d5fb4870a9 namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 6cbaaaba-200f-4ca7-a2bb-3ba9ae6910fc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 1f71a387-f1bd-44a2-aaca-93dabd5284c9 kind: Event lastTimestamp: "2026-07-09T09:45:12Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-09T09:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:12Z" name: isvc-sklearn-graph-2.18c095f062e143c8 namespace: kserve-ci-e2e-test resourceVersion: "15786" uid: b7f160d0-0670-40f5-93bd-afad09f1b4b9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:43:18.667774Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-4cp66 to ip-10-0-135-204.ec2.internal metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:43:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095d5f87627a4 namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: ec3cacdc-7cc2-4978-b9ca-c0603c8acb70 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095d602ca05f0 namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: 136fcb5a-827d-4601-b283-0d07a8610118 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:43:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095d635e6382f namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 7a012371-c408-40e1-a602-791362c7bb89 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5b65173770308fe15aad6e4df3359ab251f5bea4d39d3b1da002eeb1d533adbb" already present on machine metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095d63766031d namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: 2eb9d898-4c64-4221-97c6-a926e4559c14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095d63aba2798 namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: 636ba91b-c8c1-457b-9126-3c7ee54770be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095d63b411e42 namespace: kserve-ci-e2e-test resourceVersion: "14575" uid: 8e81f81e-d983-4e2a-a715-e9c45b27d979 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:23Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-09T09:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:23Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095d72ac783d2 namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 920a936f-5ed3-443a-9cf6-b15c3414f5a5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:52Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 28.329s (28.329s including waiting). Image size: 1305547515 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095ddc36031eb namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: 790b41e2-6a3e-42be-a8eb-a1a7e9c50839 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095ddc8e9f266 namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: eb56c870-2a67-44e1-886a-ac95e0a14f7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095ddca9a968f namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: 24cdd4cc-e62f-45f8-81a0-c532198b35a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:52Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T09:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095ddcaa956b3 namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: ebca2faa-fb9f-4730-9ffa-c7da35fc330a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.54s (2.54s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095de621ae736 namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: d1429b8c-5ea4-41b4-be04-34ce1102de9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095de67c561a5 namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 4ca89f02-6bdc-4139-ab1b-3fc5c8e5ca5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:43:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095de682db561 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: f355710d-0649-47bd-8c89-8a6699f75617 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-09T09:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:45:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095de6f3b5e6d namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 3418abc8-cb0c-497f-a552-face8a27e9a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095f8d551fabb namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: e8f587dd-a817-405d-8811-29c295f4993a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095f8d5530cf3 namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: b015f078-1a7d-4b43-ab10-92886b88b51c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4cp66 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 953cb958-4294-4c91-912a-3d4ac0bf4c67 kind: Event lastTimestamp: "2026-07-09T09:45:51Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-4cp66.18c095f971acbd7e namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: 93175a94-0135-4228-8b03-6da52606f69c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-204.ec2.internal source: component: kubelet host: ip-10-0-135-204.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14474" uid: 19fea903-c709-47c4-865d-cb761a2375fa kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-4cp66' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c095d5f7d2a078 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 2f652a13-3e56-4ad7-b425-d1519e56b9e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: cadc97f1-372d-4e76-ab3a-6d626fede72f kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: isvc-xgboost-graph-predictor.18c095d5f52a9800 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 72659c65-553a-4705-8aa5-afb014013c9e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: e57d5174-117b-4614-8d11-70ff2bf2ac82 kind: Event lastTimestamp: "2026-07-09T09:44:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:03Z" name: isvc-xgboost-graph-predictor.18c095d973d097dd namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 64df02f6-7f90-42d0-a6c0-606578d74ba1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: e57d5174-117b-4614-8d11-70ff2bf2ac82 kind: Event lastTimestamp: "2026-07-09T09:44:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:03Z" name: isvc-xgboost-graph-predictor.18c095d97444045e namespace: kserve-ci-e2e-test resourceVersion: "15007" uid: c6a8d502-71f4-4d0b-a16f-363452507afa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: e57d5174-117b-4614-8d11-70ff2bf2ac82 kind: Event lastTimestamp: "2026-07-09T09:45:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:18Z" name: isvc-xgboost-graph-predictor.18c095e3f09ff0d8 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 7f41b3c8-69cf-41f1-a5da-6e066dda5937 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T09:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: e57d5174-117b-4614-8d11-70ff2bf2ac82 kind: Event lastTimestamp: "2026-07-09T09:45:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:18Z" name: isvc-xgboost-graph-predictor.18c095e3f0ec0050 namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 91f77893-2fee-4228-8eef-c044ac8ebe61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 9912aad0-10fc-43be-a884-d03341cb2743 kind: Event lastTimestamp: "2026-07-09T09:44:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:51Z" name: isvc-xgboost-graph.18c095d5f53090ad namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6d01a29b-1df9-47ce-98a1-fcc2fb738f3f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: 9912aad0-10fc-43be-a884-d03341cb2743 kind: Event lastTimestamp: "2026-07-09T09:44:51Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-09T09:44:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:51Z" name: isvc-xgboost-graph.18c095eb7efecc80 namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: da7a6cc5-5c05-4741-be0b-e80081e04003 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:45:38.223885Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-745f4f776f-jldbm namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: e70daffd-8b6b-4248-9580-d442fb7fe972 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-745f4f776f-jldbm to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:45:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:45:38Z" name: model-chainer-745f4f776f-jldbm.18c095f676a76b0a namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: df55ae26-ca4f-4504-ad85-bc9c3416a8e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:38Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-745f4f776f-jldbm namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: e70daffd-8b6b-4248-9580-d442fb7fe972 kind: Event lastTimestamp: "2026-07-09T09:45:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:38Z" name: model-chainer-745f4f776f-jldbm.18c095f686e86f48 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: ab0bbc75-a541-4491-9b5c-474237d4649b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-745f4f776f-jldbm namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: e70daffd-8b6b-4248-9580-d442fb7fe972 kind: Event lastTimestamp: "2026-07-09T09:45:39Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:45:39Z" name: model-chainer-745f4f776f-jldbm.18c095f6b3cf64f9 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: e60f69be-9730-40d7-9c39-a562f245b3d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-745f4f776f-jldbm namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: e70daffd-8b6b-4248-9580-d442fb7fe972 kind: Event lastTimestamp: "2026-07-09T09:45:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T09:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:39Z" name: model-chainer-745f4f776f-jldbm.18c095f6b519c1a9 namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 40416569-b922-4a4d-9acc-ab919bd0bac3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-745f4f776f-jldbm namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: e70daffd-8b6b-4248-9580-d442fb7fe972 kind: Event lastTimestamp: "2026-07-09T09:45:39Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-09T09:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:39Z" name: model-chainer-745f4f776f-jldbm.18c095f6b8260f85 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: ea0fc225-8343-4390-bba6-ddcac393f350 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-745f4f776f-jldbm namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: e70daffd-8b6b-4248-9580-d442fb7fe972 kind: Event lastTimestamp: "2026-07-09T09:45:39Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-09T09:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:39Z" name: model-chainer-745f4f776f-jldbm.18c095f6b8951330 namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: 5951f896-3644-4cc8-affe-7cbfa1cb29dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-745f4f776f-jldbm namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: e70daffd-8b6b-4248-9580-d442fb7fe972 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:48Z" name: model-chainer-745f4f776f-jldbm.18c095f8d08a0b52 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 34d5aadc-f157-4d48-97f9-680a4beb11b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-745f4f776f-jldbm namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: e70daffd-8b6b-4248-9580-d442fb7fe972 kind: Event lastTimestamp: "2026-07-09T09:46:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T09:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:15Z" name: model-chainer-745f4f776f-jldbm.18c095f961bd226e namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: 5f7bd62a-a92a-4f44-9302-fad2279ad69a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-745f4f776f namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: eb9cbf2c-529a-4280-b1bd-64614c8ee86c kind: Event lastTimestamp: "2026-07-09T09:45:38Z" message: 'Created pod: model-chainer-745f4f776f-jldbm' metadata: creationTimestamp: "2026-07-09T09:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:38Z" name: model-chainer-745f4f776f.18c095f6763652f7 namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 4f81559a-c107-400a-9848-d31a193bb193 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: 395a13ec-3f4c-49b0-becf-1ac220599b5a kind: Event lastTimestamp: "2026-07-09T09:45:38Z" message: Scaled up replica set model-chainer-745f4f776f from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:38Z" name: model-chainer.18c095f6756b5a11 namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 906dfe1a-cc26-4943-96f1-fa0fb933890c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 9f9d3d6b-962f-40cf-8220-031af1638b99 kind: Event lastTimestamp: "2026-07-09T09:45:38Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-09T09:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:38Z" name: model-chainer.18c095f67a85cf07 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 1660d63a-a07a-4dad-9fbc-e7798c5213ac reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:57:13.657206Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2cd6b-748d5dcdf4-8jbw4 namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: abc57c84-5d6a-4fdc-afe6-c359baf509ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-2cd6b-748d5dcdf4-8jbw4 to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:57:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:57:13Z" name: sequence-graph-2cd6b-748d5dcdf4-8jbw4.18c0969861b5d4fc namespace: kserve-ci-e2e-test resourceVersion: "22095" uid: 4c316b5f-9bd0-4e3c-b777-68034cd461d5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:57:14Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2cd6b-748d5dcdf4-8jbw4 namespace: kserve-ci-e2e-test resourceVersion: "22099" uid: abc57c84-5d6a-4fdc-afe6-c359baf509ec kind: Event lastTimestamp: "2026-07-09T09:57:14Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:57:14Z" name: sequence-graph-2cd6b-748d5dcdf4-8jbw4.18c096987b7ff7f7 namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: 784406e8-b905-4d1a-b00a-3fcde21359fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2cd6b} kind: Pod name: sequence-graph-2cd6b-748d5dcdf4-8jbw4 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: abc57c84-5d6a-4fdc-afe6-c359baf509ec kind: Event lastTimestamp: "2026-07-09T09:57:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T09:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:57:14Z" name: sequence-graph-2cd6b-748d5dcdf4-8jbw4.18c096987ce2c96e namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: 1aedde2e-032d-4013-a263-446bd8c4a68d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2cd6b} kind: Pod name: sequence-graph-2cd6b-748d5dcdf4-8jbw4 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: abc57c84-5d6a-4fdc-afe6-c359baf509ec kind: Event lastTimestamp: "2026-07-09T09:57:14Z" message: 'Created container: sequence-graph-2cd6b' metadata: creationTimestamp: "2026-07-09T09:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:57:14Z" name: sequence-graph-2cd6b-748d5dcdf4-8jbw4.18c096987f9b768b namespace: kserve-ci-e2e-test resourceVersion: "22114" uid: 3372333b-d8d1-4883-9d74-b0969260ef10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2cd6b} kind: Pod name: sequence-graph-2cd6b-748d5dcdf4-8jbw4 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: abc57c84-5d6a-4fdc-afe6-c359baf509ec kind: Event lastTimestamp: "2026-07-09T09:57:14Z" message: Started container sequence-graph-2cd6b metadata: creationTimestamp: "2026-07-09T09:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:57:14Z" name: sequence-graph-2cd6b-748d5dcdf4-8jbw4.18c0969880083d57 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: a0486446-4d44-4a67-abbc-04eeb799c71c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2cd6b} kind: Pod name: sequence-graph-2cd6b-748d5dcdf4-8jbw4 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: abc57c84-5d6a-4fdc-afe6-c359baf509ec kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: Stopping container sequence-graph-2cd6b metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:28Z" name: sequence-graph-2cd6b-748d5dcdf4-8jbw4.18c0970b91215909 namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 395dff1a-17d0-422c-8acd-3caa1649e08f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2cd6b} kind: Pod name: sequence-graph-2cd6b-748d5dcdf4-8jbw4 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: abc57c84-5d6a-4fdc-afe6-c359baf509ec kind: Event lastTimestamp: "2026-07-09T10:05:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T10:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:55Z" name: sequence-graph-2cd6b-748d5dcdf4-8jbw4.18c0970c2765cd6a namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: d9858d53-bc4e-4916-9e74-bb595b5928ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:57:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-2cd6b-748d5dcdf4 namespace: kserve-ci-e2e-test resourceVersion: "22087" uid: 63609ca2-ac13-4c5c-882c-0cc99de6c41b kind: Event lastTimestamp: "2026-07-09T09:57:13Z" message: 'Created pod: sequence-graph-2cd6b-748d5dcdf4-8jbw4' metadata: creationTimestamp: "2026-07-09T09:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:13Z" name: sequence-graph-2cd6b-748d5dcdf4.18c09698613ea885 namespace: kserve-ci-e2e-test resourceVersion: "22093" uid: cb33fced-5d50-4b4f-b960-9ab4149bfbec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:57:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: 8c1f3cf0-3c1f-462d-be1f-a67c1b8b1a05 kind: Event lastTimestamp: "2026-07-09T09:57:13Z" message: Scaled up replica set sequence-graph-2cd6b-748d5dcdf4 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:13Z" name: sequence-graph-2cd6b.18c0969860525a08 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: 883d109e-f494-4514-a74e-85844b7b4e2c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:57:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: 9ecf06d9-f821-4801-904e-915407d5ac9f kind: Event lastTimestamp: "2026-07-09T09:57:13Z" message: InferenceGraph [sequence-graph-2cd6b] is Ready metadata: creationTimestamp: "2026-07-09T09:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:57:13Z" name: sequence-graph-2cd6b.18c096986578e7f4 namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: ee24f344-c74e-455d-ae70-83da8ce71e30 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:57:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: ce4f5e4c-f31c-4c8c-906d-77c4ce4902b4 kind: Event lastTimestamp: "2026-07-09T09:57:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:57:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:43Z" name: sequence-graph-2cd6b.18c0969bdf931f71 namespace: kserve-ci-e2e-test resourceVersion: "22334" uid: 848f7b74-f7df-4bc6-bf8c-99557eea1477 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:57:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: ce4f5e4c-f31c-4c8c-906d-77c4ce4902b4 kind: Event lastTimestamp: "2026-07-09T09:57:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:57:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:43Z" name: sequence-graph-2cd6b.18c0969be0240226 namespace: kserve-ci-e2e-test resourceVersion: "22335" uid: eff972f0-ed7a-4909-a35c-aa30d8d1b4e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:46:48.547105Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8b307-86775f9586-b64sl namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 4412a1d7-206b-4ee4-ac75-5368eeb73be8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8b307-86775f9586-b64sl to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:46:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:46:48Z" name: sequence-graph-8b307-86775f9586-b64sl.18c09606d63e98ca namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: b0d8dbb4-f6af-4618-a757-46aaf13bc4c2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:48Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8b307-86775f9586-b64sl namespace: kserve-ci-e2e-test resourceVersion: "16835" uid: 4412a1d7-206b-4ee4-ac75-5368eeb73be8 kind: Event lastTimestamp: "2026-07-09T09:46:48Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:46:48Z" name: sequence-graph-8b307-86775f9586-b64sl.18c09606ef9840eb namespace: kserve-ci-e2e-test resourceVersion: "16846" uid: 2ff7c399-b55d-42f8-b2d6-ac8c33022f6a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b307} kind: Pod name: sequence-graph-8b307-86775f9586-b64sl namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 4412a1d7-206b-4ee4-ac75-5368eeb73be8 kind: Event lastTimestamp: "2026-07-09T09:46:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T09:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:48Z" name: sequence-graph-8b307-86775f9586-b64sl.18c09606f102bede namespace: kserve-ci-e2e-test resourceVersion: "16848" uid: ab2f9841-7246-4fc0-8999-f6da29e17248 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b307} kind: Pod name: sequence-graph-8b307-86775f9586-b64sl namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 4412a1d7-206b-4ee4-ac75-5368eeb73be8 kind: Event lastTimestamp: "2026-07-09T09:46:49Z" message: 'Created container: sequence-graph-8b307' metadata: creationTimestamp: "2026-07-09T09:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:49Z" name: sequence-graph-8b307-86775f9586-b64sl.18c09606f3c75bac namespace: kserve-ci-e2e-test resourceVersion: "16850" uid: ed8ab3a6-dfbb-4d47-9d50-700f20f301fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b307} kind: Pod name: sequence-graph-8b307-86775f9586-b64sl namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 4412a1d7-206b-4ee4-ac75-5368eeb73be8 kind: Event lastTimestamp: "2026-07-09T09:46:49Z" message: Started container sequence-graph-8b307 metadata: creationTimestamp: "2026-07-09T09:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:49Z" name: sequence-graph-8b307-86775f9586-b64sl.18c09606f48da71c namespace: kserve-ci-e2e-test resourceVersion: "16851" uid: 457311f8-2b76-4d15-a94a-84286d5d8207 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b307} kind: Pod name: sequence-graph-8b307-86775f9586-b64sl namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 4412a1d7-206b-4ee4-ac75-5368eeb73be8 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Stopping container sequence-graph-8b307 metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: sequence-graph-8b307-86775f9586-b64sl.18c09679fde8bec1 namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 0411827f-3d43-4253-9753-89d74138ec50 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b307} kind: Pod name: sequence-graph-8b307-86775f9586-b64sl namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 4412a1d7-206b-4ee4-ac75-5368eeb73be8 kind: Event lastTimestamp: "2026-07-09T09:55:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T09:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:30Z" name: sequence-graph-8b307-86775f9586-b64sl.18c0967a71616c39 namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 008bbda5-11d1-4acf-b6e9-05de9e92eec1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8b307-86775f9586 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 1b2db4fb-3bd7-4793-8f31-875bc5d6df09 kind: Event lastTimestamp: "2026-07-09T09:46:48Z" message: 'Created pod: sequence-graph-8b307-86775f9586-b64sl' metadata: creationTimestamp: "2026-07-09T09:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:48Z" name: sequence-graph-8b307-86775f9586.18c09606d57d58be namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 6a625899-6b04-4924-9d43-4fbd3952d428 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: dfd85a6b-98ee-415a-8a40-45f13b11b44b kind: Event lastTimestamp: "2026-07-09T09:46:48Z" message: Scaled up replica set sequence-graph-8b307-86775f9586 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:48Z" name: sequence-graph-8b307.18c09606d4bc4120 namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: e3b03a26-152b-40a9-b41b-19dc87a3622a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: 7591dcd4-c2fa-4ad9-8dee-a5faaa00672d kind: Event lastTimestamp: "2026-07-09T09:46:48Z" message: InferenceGraph [sequence-graph-8b307] is Ready metadata: creationTimestamp: "2026-07-09T09:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:46:48Z" name: sequence-graph-8b307.18c09606da854170 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: f0b14609-7575-4155-a32b-bb099cd186a8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:47:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 6a85684d-158d-4740-936f-cadef54ec40d kind: Event lastTimestamp: "2026-07-09T09:47:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:18Z" name: sequence-graph-8b307.18c0960a532103dc namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: be86898d-8b37-4940-b1f4-149b5ee94bf9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:47:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 6a85684d-158d-4740-936f-cadef54ec40d kind: Event lastTimestamp: "2026-07-09T09:47:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:18Z" name: sequence-graph-8b307.18c0960a538c7a7a namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: 817e8463-cb96-4a2d-8de1-4ec80a3c33b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:56:03.326118Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3d13-549b5f46dd-xcdvl namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 84678253-d6d6-45ec-a027-1c7160071e8e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a3d13-549b5f46dd-xcdvl to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:56:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:56:03Z" name: sequence-graph-a3d13-549b5f46dd-xcdvl.18c0968801a693c7 namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: cc1b6949-8f16-464c-9cac-06d9a865e73d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3d13-549b5f46dd-xcdvl namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: 84678253-d6d6-45ec-a027-1c7160071e8e kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:56:03Z" name: sequence-graph-a3d13-549b5f46dd-xcdvl.18c096881b13f618 namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: 53543698-98cb-4432-a518-9156058f3926 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3d13} kind: Pod name: sequence-graph-a3d13-549b5f46dd-xcdvl namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: 84678253-d6d6-45ec-a027-1c7160071e8e kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T09:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:03Z" name: sequence-graph-a3d13-549b5f46dd-xcdvl.18c096881c6b4b17 namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: 1cba643b-26ba-4480-bfdf-266844fe0790 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3d13} kind: Pod name: sequence-graph-a3d13-549b5f46dd-xcdvl namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: 84678253-d6d6-45ec-a027-1c7160071e8e kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: 'Created container: sequence-graph-a3d13' metadata: creationTimestamp: "2026-07-09T09:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:03Z" name: sequence-graph-a3d13-549b5f46dd-xcdvl.18c096881fa0da4c namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 3b7ce051-a519-4cdc-9b0d-525112cb3672 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3d13} kind: Pod name: sequence-graph-a3d13-549b5f46dd-xcdvl namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: 84678253-d6d6-45ec-a027-1c7160071e8e kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: Started container sequence-graph-a3d13 metadata: creationTimestamp: "2026-07-09T09:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:03Z" name: sequence-graph-a3d13-549b5f46dd-xcdvl.18c09688200b5c32 namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: d1a8978e-9ed5-4c67-87bf-a4dfbc1008fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3d13} kind: Pod name: sequence-graph-a3d13-549b5f46dd-xcdvl namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: 84678253-d6d6-45ec-a027-1c7160071e8e kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: Stopping container sequence-graph-a3d13 metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:13Z" name: sequence-graph-a3d13-549b5f46dd-xcdvl.18c0968a5a45a718 namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 6bed368a-450d-47db-b754-d296c99b3a7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3d13} kind: Pod name: sequence-graph-a3d13-549b5f46dd-xcdvl namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: 84678253-d6d6-45ec-a027-1c7160071e8e kind: Event lastTimestamp: "2026-07-09T09:56:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T09:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:40Z" name: sequence-graph-a3d13-549b5f46dd-xcdvl.18c0968adb95eb06 namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: 597e71c9-cb87-4e07-8bdb-c96b377ccfe5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a3d13-549b5f46dd namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: c6336c87-c5c3-4503-9ae6-1ced110519c3 kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: 'Created pod: sequence-graph-a3d13-549b5f46dd-xcdvl' metadata: creationTimestamp: "2026-07-09T09:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:03Z" name: sequence-graph-a3d13-549b5f46dd.18c0968801408e0c namespace: kserve-ci-e2e-test resourceVersion: "21285" uid: 07c2483f-23c2-4edf-bf31-909af1a195b4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a3d13 namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: ca1879d1-95ff-4ff7-b18e-362babaf97bf kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: Scaled up replica set sequence-graph-a3d13-549b5f46dd from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:03Z" name: sequence-graph-a3d13.18c0968800282042 namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: a11a972e-b74a-4bcb-a546-2c393cc098d5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a3d13 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: dafa91bd-0591-4d2c-9945-77c9ddc69b27 kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: InferenceGraph [sequence-graph-a3d13] is Ready metadata: creationTimestamp: "2026-07-09T09:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:03Z" name: sequence-graph-a3d13.18c0968805caa007 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: 27820fab-db8d-4c51-a995-af54649fdda2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T10:05:32.613277Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9d678-56c66cc487-c4m49 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 5185756b-f66b-43b8-abec-4422ec5c999e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-9d678-56c66cc487-c4m49 to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T10:05:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T10:05:32Z" name: splitter-graph-9d678-56c66cc487-c4m49.18c0970c8dcf34fb namespace: kserve-ci-e2e-test resourceVersion: "26177" uid: fe7d8d08-1056-4a8a-8715-439eadb52049 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:33Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9d678-56c66cc487-c4m49 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: 5185756b-f66b-43b8-abec-4422ec5c999e kind: Event lastTimestamp: "2026-07-09T10:05:33Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T10:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T10:05:33Z" name: splitter-graph-9d678-56c66cc487-c4m49.18c0970ca752a472 namespace: kserve-ci-e2e-test resourceVersion: "26200" uid: ddbe91d3-6f40-4b66-b5d8-f90c9915dd4f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9d678} kind: Pod name: splitter-graph-9d678-56c66cc487-c4m49 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 5185756b-f66b-43b8-abec-4422ec5c999e kind: Event lastTimestamp: "2026-07-09T10:05:33Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T10:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:33Z" name: splitter-graph-9d678-56c66cc487-c4m49.18c0970ca8a0d4dc namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: a54221ec-2be9-476f-97fd-6adb9022f59f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9d678} kind: Pod name: splitter-graph-9d678-56c66cc487-c4m49 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 5185756b-f66b-43b8-abec-4422ec5c999e kind: Event lastTimestamp: "2026-07-09T10:05:33Z" message: 'Created container: splitter-graph-9d678' metadata: creationTimestamp: "2026-07-09T10:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:33Z" name: splitter-graph-9d678-56c66cc487-c4m49.18c0970cab8adff7 namespace: kserve-ci-e2e-test resourceVersion: "26203" uid: d38effcc-5916-4f07-ae83-f158895ae6a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9d678} kind: Pod name: splitter-graph-9d678-56c66cc487-c4m49 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 5185756b-f66b-43b8-abec-4422ec5c999e kind: Event lastTimestamp: "2026-07-09T10:05:33Z" message: Started container splitter-graph-9d678 metadata: creationTimestamp: "2026-07-09T10:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:33Z" name: splitter-graph-9d678-56c66cc487-c4m49.18c0970cabf91387 namespace: kserve-ci-e2e-test resourceVersion: "26204" uid: 1754bc14-f30b-4051-81da-caefc3bec7e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9d678} kind: Pod name: splitter-graph-9d678-56c66cc487-c4m49 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 5185756b-f66b-43b8-abec-4422ec5c999e kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: Stopping container splitter-graph-9d678 metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:42Z" name: splitter-graph-9d678-56c66cc487-c4m49.18c0970ee7353047 namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 2875b09f-7133-4919-8c61-4b8c9bb17b5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9d678} kind: Pod name: splitter-graph-9d678-56c66cc487-c4m49 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 5185756b-f66b-43b8-abec-4422ec5c999e kind: Event lastTimestamp: "2026-07-09T10:06:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:09Z" name: splitter-graph-9d678-56c66cc487-c4m49.18c0970f3d766816 namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 0a464472-52b9-4dd1-9381-945c40028e59 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-9d678-56c66cc487 namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: 12f40dae-8b9f-4dea-9e62-be11dde84622 kind: Event lastTimestamp: "2026-07-09T10:05:32Z" message: 'Created pod: splitter-graph-9d678-56c66cc487-c4m49' metadata: creationTimestamp: "2026-07-09T10:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:32Z" name: splitter-graph-9d678-56c66cc487.18c0970c8d6c136d namespace: kserve-ci-e2e-test resourceVersion: "26172" uid: 9eaf1e2d-7ee8-4af1-ad02-861e86ca51b2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-9d678 namespace: kserve-ci-e2e-test resourceVersion: "26165" uid: 177d8e33-2ae8-4722-866e-37fe8532803e kind: Event lastTimestamp: "2026-07-09T10:05:32Z" message: Scaled up replica set splitter-graph-9d678-56c66cc487 from 0 to 1 metadata: creationTimestamp: "2026-07-09T10:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:32Z" name: splitter-graph-9d678.18c0970c8cba8442 namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 384e052d-2d34-40ba-8937-6b551cd05054 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9d678 namespace: kserve-ci-e2e-test resourceVersion: "26185" uid: e2efcdde-b485-4274-85f3-7b209cccccde kind: Event lastTimestamp: "2026-07-09T10:05:32Z" message: InferenceGraph [splitter-graph-9d678] is Ready metadata: creationTimestamp: "2026-07-09T10:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:32Z" name: splitter-graph-9d678.18c0970c91cfa7f5 namespace: kserve-ci-e2e-test resourceVersion: "26187" uid: 866a8b11-2a87-453a-a54e-41cd0034ad51 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9d678 namespace: kserve-ci-e2e-test resourceVersion: "26159" uid: e2efcdde-b485-4274-85f3-7b209cccccde kind: Event lastTimestamp: "2026-07-09T10:05:32Z" message: 'Failed to update status for InferenceGraph "splitter-graph-9d678": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-9d678": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:32Z" name: splitter-graph-9d678.18c0970c944dfea6 namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: 9bcb461a-b313-4439-aba6-3aa18ca95f3b reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9d678 namespace: kserve-ci-e2e-test resourceVersion: "26159" uid: e2efcdde-b485-4274-85f3-7b209cccccde kind: Event lastTimestamp: "2026-07-09T10:05:32Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-9d678": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:32Z" name: splitter-graph-9d678.18c0970c944e3f84 namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: 763d18c5-7bd4-4c75-b647-43bd6a7280b0 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T10:06:42.949688Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a7fc2-5d85f6c84f-hxbwt namespace: kserve-ci-e2e-test resourceVersion: "26993" uid: 9df3cb7b-0fe3-4690-8692-c19f344ef5a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a7fc2-5d85f6c84f-hxbwt to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T10:06:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T10:06:42Z" name: splitter-graph-a7fc2-5d85f6c84f-hxbwt.18c0971cee2fb2b6 namespace: kserve-ci-e2e-test resourceVersion: "26999" uid: 15cbff1f-ce32-4d99-a336-ae87f44b0442 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:43Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a7fc2-5d85f6c84f-hxbwt namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: 9df3cb7b-0fe3-4690-8692-c19f344ef5a6 kind: Event lastTimestamp: "2026-07-09T10:06:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-a7fc2-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T10:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:43Z" name: splitter-graph-a7fc2-5d85f6c84f-hxbwt.18c0971cf8de5cc9 namespace: kserve-ci-e2e-test resourceVersion: "27012" uid: 7d4506b0-9584-470e-97e6-046a8c71f961 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:43Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a7fc2-5d85f6c84f-hxbwt namespace: kserve-ci-e2e-test resourceVersion: "27002" uid: 9df3cb7b-0fe3-4690-8692-c19f344ef5a6 kind: Event lastTimestamp: "2026-07-09T10:06:43Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T10:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T10:06:43Z" name: splitter-graph-a7fc2-5d85f6c84f-hxbwt.18c0971d2b98c94b namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: af995f02-d720-49d1-8e7f-ea68bdb905dc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a7fc2} kind: Pod name: splitter-graph-a7fc2-5d85f6c84f-hxbwt namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: 9df3cb7b-0fe3-4690-8692-c19f344ef5a6 kind: Event lastTimestamp: "2026-07-09T10:06:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T10:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:44Z" name: splitter-graph-a7fc2-5d85f6c84f-hxbwt.18c0971d2d021078 namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: e33a87f3-e12a-4f3f-b52a-6fc3e356c599 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a7fc2} kind: Pod name: splitter-graph-a7fc2-5d85f6c84f-hxbwt namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: 9df3cb7b-0fe3-4690-8692-c19f344ef5a6 kind: Event lastTimestamp: "2026-07-09T10:06:44Z" message: 'Created container: splitter-graph-a7fc2' metadata: creationTimestamp: "2026-07-09T10:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:44Z" name: splitter-graph-a7fc2-5d85f6c84f-hxbwt.18c0971d305de809 namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: 864b28ae-ea54-44f1-9ea4-9ed52d6bcd8d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a7fc2} kind: Pod name: splitter-graph-a7fc2-5d85f6c84f-hxbwt namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: 9df3cb7b-0fe3-4690-8692-c19f344ef5a6 kind: Event lastTimestamp: "2026-07-09T10:06:44Z" message: Started container splitter-graph-a7fc2 metadata: creationTimestamp: "2026-07-09T10:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:44Z" name: splitter-graph-a7fc2-5d85f6c84f-hxbwt.18c0971d30ce83ea namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: 4d1701a5-92c9-4845-a39a-3e2a14160078 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a7fc2} kind: Pod name: splitter-graph-a7fc2-5d85f6c84f-hxbwt namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: 9df3cb7b-0fe3-4690-8692-c19f344ef5a6 kind: Event lastTimestamp: "2026-07-09T10:14:57Z" message: Stopping container splitter-graph-a7fc2 metadata: creationTimestamp: "2026-07-09T10:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:14:57Z" name: splitter-graph-a7fc2-5d85f6c84f-hxbwt.18c09790196d5106 namespace: kserve-ci-e2e-test resourceVersion: "30502" uid: d691638b-3a45-4ee6-9dfb-c2f4bab2bfce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a7fc2} kind: Pod name: splitter-graph-a7fc2-5d85f6c84f-hxbwt namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: 9df3cb7b-0fe3-4690-8692-c19f344ef5a6 kind: Event lastTimestamp: "2026-07-09T10:15:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T10:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:15:25Z" name: splitter-graph-a7fc2-5d85f6c84f-hxbwt.18c09790c53538cd namespace: kserve-ci-e2e-test resourceVersion: "30738" uid: cd55c08b-cd29-4e46-8879-5c363790799c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a7fc2-5d85f6c84f namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: 83d2bb3a-6256-491f-996e-2f7d09682574 kind: Event lastTimestamp: "2026-07-09T10:06:42Z" message: 'Created pod: splitter-graph-a7fc2-5d85f6c84f-hxbwt' metadata: creationTimestamp: "2026-07-09T10:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:42Z" name: splitter-graph-a7fc2-5d85f6c84f.18c0971ced8c55ee namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: cbe9fe52-6d53-4c45-b631-b00b045a35fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26986" uid: c3a592c6-510a-4a7d-98b2-54c52ac75c42 kind: Event lastTimestamp: "2026-07-09T10:06:42Z" message: Scaled up replica set splitter-graph-a7fc2-5d85f6c84f from 0 to 1 metadata: creationTimestamp: "2026-07-09T10:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:42Z" name: splitter-graph-a7fc2.18c0971ceccdb34a namespace: kserve-ci-e2e-test resourceVersion: "26991" uid: 7922b803-bfc7-4ce1-9fc5-f018a38fd891 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "27004" uid: ed67ee8e-0134-4001-aab9-c6f68b8d21d9 kind: Event lastTimestamp: "2026-07-09T10:06:42Z" message: InferenceGraph [splitter-graph-a7fc2] is Ready metadata: creationTimestamp: "2026-07-09T10:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:43Z" name: splitter-graph-a7fc2.18c0971cf1269057 namespace: kserve-ci-e2e-test resourceVersion: "27008" uid: ad87112f-663b-4726-9c88-81ee4c2660c2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: 4bb103f6-5efb-4aac-974d-b1d61de0a667 kind: Event lastTimestamp: "2026-07-09T10:06:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:06:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:57Z" name: splitter-graph-a7fc2.18c097206b37705a namespace: kserve-ci-e2e-test resourceVersion: "27140" uid: 9d66cd77-7903-4413-86c7-baa57eea4189 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: 4bb103f6-5efb-4aac-974d-b1d61de0a667 kind: Event lastTimestamp: "2026-07-09T10:06:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:06:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:57Z" name: splitter-graph-a7fc2.18c097206b7c4797 namespace: kserve-ci-e2e-test resourceVersion: "27142" uid: cf9ff095-32b8-4c77-a044-99816b3b4993 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:07:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: 4bb103f6-5efb-4aac-974d-b1d61de0a667 kind: Event lastTimestamp: "2026-07-09T10:07:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:07:12Z" name: splitter-graph-a7fc2.18c09723ea92b767 namespace: kserve-ci-e2e-test resourceVersion: "27245" uid: bd0a57e3-cac7-4fe2-978e-fe238a3cf3c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:07:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: 4bb103f6-5efb-4aac-974d-b1d61de0a667 kind: Event lastTimestamp: "2026-07-09T10:07:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:07:12Z" name: splitter-graph-a7fc2.18c09723eb36f119 namespace: kserve-ci-e2e-test resourceVersion: "27247" uid: 34a30c64-d629-4bec-83b0-368c64dadd12 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:43:18.453158Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1980c-predictor-8479bf99dd-f2xcf to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:43:18Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d5ebab6ba5 namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 1848c697-3554-47ff-a72d-3bbd24650b61 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1980c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:18Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d5fa0ec8c9 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 95949796-44cc-4ef2-a5ac-d490a2c8dfb3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:43:19Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d62919af14 namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: f9b0a158-f5eb-4d82-813c-bf2053513cb1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:19Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" metadata: creationTimestamp: "2026-07-09T09:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:19Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d62a730156 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: b90d923c-31e2-4c44-9b17-2dcb6b79d538 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:32Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" in 12.9s (12.9s including waiting). Image size: 1335676005 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:32Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d92b61c975 namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: 404f4881-b0fb-4515-828e-64920b440ee1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:32Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d92e8154fa namespace: kserve-ci-e2e-test resourceVersion: "14692" uid: 07b18b07-c3eb-41fe-8be9-6446a7d7345b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:32Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d92ef63989 namespace: kserve-ci-e2e-test resourceVersion: "14693" uid: 1332c8e8-8890-4a73-83d9-d819c8cc32fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T09:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:32Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d92f042f81 namespace: kserve-ci-e2e-test resourceVersion: "14694" uid: e4120665-1555-428c-ba0b-b3545500232c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:34Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.415s (2.415s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T09:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:34Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d9bef93753 namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 7c79e90a-386c-4721-b05e-6f95c8a52d9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:34Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d9c244b687 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 72c53108-d906-4a20-8b9d-9608e848cae9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:43:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:43:34Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d9c2bce01e namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 95c1b2df-fe27-4cca-a65c-51932d6d5e70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:44:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:11Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095d9dd3e8de2 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: eb56fe50-23e9-483b-9cce-f5288561ac81 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:52Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095ebd2e3d325 namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 1fcc2597-137f-4505-b7d2-09342d00d122 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 852da3a5-11db-4411-8a3f-b1a16c31d218 kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:52Z" name: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf.18c095ebd2e45ec9 namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: 70954e96-9967-4bd6-9659-d5c25179ee16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1980c-predictor-8479bf99dd namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: 9848184d-8521-427a-a44b-adaccdfe158c kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: 'Created pod: success-200-isvc-1980c-predictor-8479bf99dd-f2xcf' metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: success-200-isvc-1980c-predictor-8479bf99dd.18c095d5eb3c0d64 namespace: kserve-ci-e2e-test resourceVersion: "14428" uid: 5004dab5-773d-45c6-bfcf-4bb36e7dedd1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14417" uid: 5628a080-298e-4b26-87aa-112e1032af77 kind: Event lastTimestamp: "2026-07-09T09:43:18Z" message: Scaled up replica set success-200-isvc-1980c-predictor-8479bf99dd from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:43:18Z" name: success-200-isvc-1980c-predictor.18c095d5e98d3ef7 namespace: kserve-ci-e2e-test resourceVersion: "14420" uid: 1dd1ba06-ce2f-42c0-934a-445a83ab388d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 5ba3dba5-bb65-43ef-8e54-e873555f21d7 kind: Event lastTimestamp: "2026-07-09T09:44:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:03Z" name: success-200-isvc-1980c-predictor.18c095d968ebe539 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 533bb493-c081-4aa8-a9fd-bb5c0239122d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:43:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 5ba3dba5-bb65-43ef-8e54-e873555f21d7 kind: Event lastTimestamp: "2026-07-09T09:44:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:03Z" name: success-200-isvc-1980c-predictor.18c095d9697dea25 namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: d13d4f29-9ded-4f59-a7b7-32cc3e8ef18b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: 5ba3dba5-bb65-43ef-8e54-e873555f21d7 kind: Event lastTimestamp: "2026-07-09T09:44:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:48Z" name: success-200-isvc-1980c-predictor.18c095e3e679280a namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: ec70a6d5-94ec-4401-b5b9-b71053ccfd84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1980c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: 5ba3dba5-bb65-43ef-8e54-e873555f21d7 kind: Event lastTimestamp: "2026-07-09T09:44:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:48Z" name: success-200-isvc-1980c-predictor.18c095e3e6eb9e25 namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: 9ebbba64-78e9-4516-837b-0b4d6405be27 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:43:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1980c namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: ebdf073f-7a84-4535-974c-cc484592bf23 kind: Event lastTimestamp: "2026-07-09T09:44:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:21Z" name: success-200-isvc-1980c.18c095d5e9a864e8 namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: 3d7fbf01-908a-4288-958f-c21708f74144 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1980c namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: ebdf073f-7a84-4535-974c-cc484592bf23 kind: Event lastTimestamp: "2026-07-09T09:44:21Z" message: InferenceService [success-200-isvc-1980c] is Ready metadata: creationTimestamp: "2026-07-09T09:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:21Z" name: success-200-isvc-1980c.18c095e4985809d1 namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: 3d2a44ba-b3c1-4a3c-85f5-a3731abceed6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:56:13.578552Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2cd6b-predictor-84c765884d-xbxxn to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:56:13Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968a64be4af8 namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 5a7b32ed-beeb-448b-be00-2f145f12f128 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2cd6b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:13Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968a72402e05 namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 28be40f4-05ec-40d5-8da7-9d7fe5dceec9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:56:14Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968aa303ad5e namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: feaf4423-0b4f-45f4-b130-1df7bc3201cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968aa477c836 namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 55ee5913-ef73-4526-94d5-e7aa03cf36b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968aa7a8bcdd namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: ed586167-38fc-4bf2-b165-1dfebf64370a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968aa822ff1b namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 416c607b-ad2c-4e48-9eb1-f101af6885af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968aa82ebbff namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: ad6982e7-a407-4ee8-b87b-86b99a7b1b9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968aab39571e namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: 862f68e8-139c-4580-91e2-2894a058f087 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T09:56:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:14Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968aaba99213 namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: e1a79784-4d82-40b2-a6dc-8386d6de8d0d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T09:56:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:52Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0968b0b194f62 namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 9e94cc54-e94c-47dd-a4b6-2ad91c44dad0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:28Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0970b9987c986 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 9f1a15dd-fba7-4209-9a21-22c08d8de511 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 0fdaa93a-c581-465a-9675-9c74c04783ba kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:28Z" name: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn.18c0970b99883f56 namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: f11f7339-7401-4879-a384-c16ba71b781f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2cd6b-predictor-84c765884d namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: ef315a53-237a-45e3-954e-ee641f560de6 kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: 'Created pod: success-200-isvc-2cd6b-predictor-84c765884d-xbxxn' metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:13Z" name: success-200-isvc-2cd6b-predictor-84c765884d.18c0968a644cc970 namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: 96312743-a655-4b66-ba98-8ac425a8df46 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 3f6404cb-36a0-4401-aa75-2265d83241dd kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: Scaled up replica set success-200-isvc-2cd6b-predictor-84c765884d from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:13Z" name: success-200-isvc-2cd6b-predictor.18c0968a62aa1102 namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: 193d2dc5-cc81-430f-9546-83635bdb5c13 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: b2a4436a-7c16-420e-939e-1471c8817cbe kind: Event lastTimestamp: "2026-07-09T09:56:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:28Z" name: success-200-isvc-2cd6b-predictor.18c0968de10e5fbd namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: 974754b9-f530-4ceb-9a54-fe6b50f14fa3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: b2a4436a-7c16-420e-939e-1471c8817cbe kind: Event lastTimestamp: "2026-07-09T09:56:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:28Z" name: success-200-isvc-2cd6b-predictor.18c0968de1570b7c namespace: kserve-ci-e2e-test resourceVersion: "21716" uid: 411e82eb-c737-4c8a-b81b-53f902127ac0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:56:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: b2a4436a-7c16-420e-939e-1471c8817cbe kind: Event lastTimestamp: "2026-07-09T09:57:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:28Z" name: success-200-isvc-2cd6b-predictor.18c09691609e30fc namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: 1f458fe8-d781-41d0-b685-ee034a83bb26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:56:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2cd6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: b2a4436a-7c16-420e-939e-1471c8817cbe kind: Event lastTimestamp: "2026-07-09T09:57:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:57:28Z" name: success-200-isvc-2cd6b-predictor.18c096916116cd96 namespace: kserve-ci-e2e-test resourceVersion: "22232" uid: b93c5347-032a-4c4e-8230-dce0481db945 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: e7fb87c0-c74b-4956-95eb-99cf9a8d16bc kind: Event lastTimestamp: "2026-07-09T09:57:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:57:02Z" name: success-200-isvc-2cd6b.18c0968a62b33911 namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: eb2667b7-cbd2-49cb-af15-d1e788010ad1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: e7fb87c0-c74b-4956-95eb-99cf9a8d16bc kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: 'Failed to update status for InferenceService "success-200-isvc-2cd6b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2cd6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:13Z" name: success-200-isvc-2cd6b.18c0968a6315513b namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: 61a3fa4a-686d-4e20-bf66-8ae6e75a39a2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: e7fb87c0-c74b-4956-95eb-99cf9a8d16bc kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2cd6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:13Z" name: success-200-isvc-2cd6b.18c0968a6315b0fb namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: a5af4731-fc1a-4d5e-badc-87f71e8c4754 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2cd6b namespace: kserve-ci-e2e-test resourceVersion: "21998" uid: e7fb87c0-c74b-4956-95eb-99cf9a8d16bc kind: Event lastTimestamp: "2026-07-09T09:57:02Z" message: InferenceService [success-200-isvc-2cd6b] is Ready metadata: creationTimestamp: "2026-07-09T09:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:57:02Z" name: success-200-isvc-2cd6b.18c09695c663139f namespace: kserve-ci-e2e-test resourceVersion: "21999" uid: 6c67ac5f-2cf6-491b-abe1-70684c83428e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:55:17.694240Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-42ebc-predictor-7944cc6db6-spf5t to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:55:17Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967d61c7b4ba namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: 404d96e7-f8ea-4814-8e26-921d481f0703 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-42ebc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:17Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967d6b8d8548 namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 78ac3fca-2825-496b-8f04-9790aa474ac7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:55:18Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967d9f12bd2e namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: ae2677a5-a253-41e1-9369-d44b2ca3d657 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967da05c431d namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: efed98fb-eab9-404b-b824-ac65e792be20 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967da41f9d9f namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: e21e3c9e-82c3-4313-851e-0720fd63c4be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967da49210d8 namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: e2475a32-3c6d-48bc-adeb-63021b3d8f0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967da49b9f69 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: a2200a72-d7af-4c9a-9a79-a55b700830d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967da82edef9 namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 400d46b7-366b-40b3-b7d5-327a83609cbd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T09:55:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:18Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967da89f8f43 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 4f9e637f-e368-4ecb-bfaa-a2905da13521 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T09:55:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T10:04:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:35Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c0967db7ea263d namespace: kserve-ci-e2e-test resourceVersion: "25404" uid: 051deab8-50ac-4840-b6fa-db2575546674 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:32Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c096fe8f3f10d8 namespace: kserve-ci-e2e-test resourceVersion: "25208" uid: 5a1d3e49-4360-4e4c-a22a-7ac5ee07fdb2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:32Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c096fe8f3fde7b namespace: kserve-ci-e2e-test resourceVersion: "25210" uid: 9a8c68e9-0cd6-4958-adc5-98ec5f3460e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: be390e27-d69e-496c-b262-531a44449945 kind: Event lastTimestamp: "2026-07-09T10:04:35Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:35Z" name: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t.18c096ff2c61f893 namespace: kserve-ci-e2e-test resourceVersion: "25403" uid: a7ea347d-3d8a-4e92-9e3c-5555f3fb7295 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-42ebc-predictor-7944cc6db6 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: df6b5785-59e3-41e4-8123-71110f044430 kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: 'Created pod: success-200-isvc-42ebc-predictor-7944cc6db6-spf5t' metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:17Z" name: success-200-isvc-42ebc-predictor-7944cc6db6.18c0967d614ce0a5 namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: cc789435-c339-4f22-b58f-8daad2762d1e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: f734e156-9af2-4b78-bb37-352df1c76278 kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: Scaled up replica set success-200-isvc-42ebc-predictor-7944cc6db6 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:17Z" name: success-200-isvc-42ebc-predictor.18c0967d5f7cf3fe namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: b6bddb04-68f9-459a-bdc2-4bb886b2b675 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: 359f5d69-420a-4c88-a492-3105475330d8 kind: Event lastTimestamp: "2026-07-09T09:55:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:32Z" name: success-200-isvc-42ebc-predictor.18c09680ddff6634 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: f28533f9-683e-4e69-92de-5393b1474403 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: 359f5d69-420a-4c88-a492-3105475330d8 kind: Event lastTimestamp: "2026-07-09T09:55:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:32Z" name: success-200-isvc-42ebc-predictor.18c09680de675200 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 9c98e39b-8d2c-4997-ac2f-516e1d94994a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:55:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: 359f5d69-420a-4c88-a492-3105475330d8 kind: Event lastTimestamp: "2026-07-09T09:56:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:32Z" name: success-200-isvc-42ebc-predictor.18c096845d7ddf51 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: a756eeee-a1ff-4b96-b2e8-0afcc79e872c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:55:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42ebc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: 359f5d69-420a-4c88-a492-3105475330d8 kind: Event lastTimestamp: "2026-07-09T09:56:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:32Z" name: success-200-isvc-42ebc-predictor.18c096845de85fe0 namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: 235a792d-2e0e-4987-8f69-b566e82010a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42ebc namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: d8f8e385-df76-456b-85be-1fb258c3e425 kind: Event lastTimestamp: "2026-07-09T09:56:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:05Z" name: success-200-isvc-42ebc.18c0967d5fada200 namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: a01b4c7a-00e7-40c2-ab84-ecc2aa3285b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42ebc namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: d8f8e385-df76-456b-85be-1fb258c3e425 kind: Event lastTimestamp: "2026-07-09T09:56:05Z" message: InferenceService [success-200-isvc-42ebc] is Ready metadata: creationTimestamp: "2026-07-09T09:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:56:05Z" name: success-200-isvc-42ebc.18c09688740ae630 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: dee92b6b-970b-4c06-a68b-38445477d67d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T10:05:28.581794Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T10:05:28Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c0970b9d83af94 namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: bd539932-e289-47a5-bf98-4990a6ace250 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26067" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T10:05:29Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c0970bb729026a namespace: kserve-ci-e2e-test resourceVersion: "26102" uid: a535cbad-fe51-4774-9f6c-8f03f98c4a78 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c0970bb895b032 namespace: kserve-ci-e2e-test resourceVersion: "26107" uid: ffefa70e-bfae-4116-995b-15dd59808777 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c0970bbb053e90 namespace: kserve-ci-e2e-test resourceVersion: "26108" uid: dc573e6e-8cd5-4dbe-af76-efe9cd995048 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c0970bbb7777d4 namespace: kserve-ci-e2e-test resourceVersion: "26109" uid: c992919d-2899-4ff9-aee4-f879fe6c01b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c0970bbb821621 namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: e0e7c8d5-422d-4646-9138-31ab2ba9e983 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c0970bbea341c8 namespace: kserve-ci-e2e-test resourceVersion: "26113" uid: 28e3f648-b0cd-44db-8748-cd859bb4fb92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:05:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:29Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c0970bbf24e82b namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: 6768eca5-1921-46eb-8046-34890c4bf60e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:06:06Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:06Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c0970c0e74e244 namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: 6bbc0431-c0e7-4da7-85ff-aeb3f47eb541 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:22:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:22:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:22:48Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c097fda43169ca namespace: kserve-ci-e2e-test resourceVersion: "33780" uid: a01452cf-eec6-4a5d-a94c-b4ccba72c613 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:22:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 58915a12-d808-4045-a40f-06eff6835839 kind: Event lastTimestamp: "2026-07-09T10:22:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:22:48Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g.18c097fda4323567 namespace: kserve-ci-e2e-test resourceVersion: "33781" uid: 9fbac66f-e7fd-41bf-97c1-9f316dc16b40 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-52f9b-predictor-9f9dc8c54 namespace: kserve-ci-e2e-test resourceVersion: "26047" uid: b1d2e3da-5f1d-4c87-bc4d-28f9a58f4c67 kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: 'Created pod: success-200-isvc-52f9b-predictor-9f9dc8c54-cs84g' metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:28Z" name: success-200-isvc-52f9b-predictor-9f9dc8c54.18c0970b9cdf3db8 namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: 50443bc3-7fb5-4038-b922-aee45873f240 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26044" uid: 4187bc46-5db1-4007-9a96-fdaea0d81456 kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: Scaled up replica set success-200-isvc-52f9b-predictor-9f9dc8c54 from 0 to 1 metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:28Z" name: success-200-isvc-52f9b-predictor.18c0970b9ae5210a namespace: kserve-ci-e2e-test resourceVersion: "26049" uid: e9191311-8044-4c46-bacf-49bd5d507bdf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 582b8719-1102-44ec-9fbb-7898a5b6d39b kind: Event lastTimestamp: "2026-07-09T10:05:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:58Z" name: success-200-isvc-52f9b-predictor.18c0970f196edd5e namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: e22c1427-8dc9-490c-8d3c-63cd28a0f7e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 582b8719-1102-44ec-9fbb-7898a5b6d39b kind: Event lastTimestamp: "2026-07-09T10:05:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:58Z" name: success-200-isvc-52f9b-predictor.18c0970f19b501a5 namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: 62046308-9df4-421c-a7fb-af4e944840bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T10:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26407" uid: 582b8719-1102-44ec-9fbb-7898a5b6d39b kind: Event lastTimestamp: "2026-07-09T10:06:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:43Z" name: success-200-isvc-52f9b-predictor.18c097161716dac7 namespace: kserve-ci-e2e-test resourceVersion: "27020" uid: 832386e9-62e4-41f6-b59c-7c3475b2c1d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T10:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52f9b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26407" uid: 582b8719-1102-44ec-9fbb-7898a5b6d39b kind: Event lastTimestamp: "2026-07-09T10:06:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:43Z" name: success-200-isvc-52f9b-predictor.18c0971617842056 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 94c775c9-6cbf-4209-b711-0ca7c9e511a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26021" uid: ae65aed2-b274-4cd1-b001-44dd7ba92438 kind: Event lastTimestamp: "2026-07-09T10:06:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:16Z" name: success-200-isvc-52f9b.18c0970b9ae82bb8 namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: 8faf2522-68b1-412d-bdc2-412d1b899ed1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26021" uid: ae65aed2-b274-4cd1-b001-44dd7ba92438 kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-52f9b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-52f9b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:28Z" name: success-200-isvc-52f9b.18c0970b9b3c1de9 namespace: kserve-ci-e2e-test resourceVersion: "26052" uid: 07f80894-f766-437e-b4af-9e0dd6c74a5f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26021" uid: ae65aed2-b274-4cd1-b001-44dd7ba92438 kind: Event lastTimestamp: "2026-07-09T10:05:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-52f9b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:28Z" name: success-200-isvc-52f9b.18c0970b9b3c5097 namespace: kserve-ci-e2e-test resourceVersion: "26053" uid: 345fc5d6-fb8b-4298-8d5e-e7cd62b0b033 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: ae65aed2-b274-4cd1-b001-44dd7ba92438 kind: Event lastTimestamp: "2026-07-09T10:06:16Z" message: InferenceService [success-200-isvc-52f9b] is Ready metadata: creationTimestamp: "2026-07-09T10:06:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:16Z" name: success-200-isvc-52f9b.18c09716c9e09b7c namespace: kserve-ci-e2e-test resourceVersion: "26722" uid: 8695bc3f-92b3-4513-90d4-2e6ba510dfe6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:45:48.528154Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8b307-predictor-94cb94958-qkdf6 to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:45:48Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f8dcd6210b namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: ac857489-e8a7-4e90-a84c-5bf4d7094bb1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8b307-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:48Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f8e858bd62 namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: 7021c05c-c5cb-4b64-92c6-ea2485602b64 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:45:49Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f91a54f223 namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 720b8bc1-7280-486a-9307-9a4a2148e86a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f91ba93936 namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 72799ac5-6a6c-4e95-980c-738bc1721ba9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f91ea01060 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: d44a8af7-64e8-4282-844c-13d06ebae007 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f91f10401c namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 2c6a8999-abed-4440-a33f-7d68edbebd33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f91f1a427e namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 0f074ec0-606d-4ae4-930f-9db6730ce6a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f921df3a85 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: 0070c32b-b461-450a-8840-7ea686737493 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:45:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:49Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f9224be6a1 namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: eaa9b463-799a-4ac1-a8fd-2762422df154 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:46:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:46:26Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c095f96c6c2c02 namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: d3cda998-e0a3-45a4-9082-eb5c77a6d72d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c0967a048f6154 namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 39c03fc8-4706-4982-a305-88d3b2121288 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b307-predictor-94cb94958-qkdf6 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 5bec4f26-b047-49c6-9a6d-71c427f0b25b kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: success-200-isvc-8b307-predictor-94cb94958-qkdf6.18c0967a048ffdaa namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: c415b2f9-72e8-4046-b60f-9ce556401cd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8b307-predictor-94cb94958 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 1670f5e7-1a17-4b42-ac22-cd2487d71c59 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: 'Created pod: success-200-isvc-8b307-predictor-94cb94958-qkdf6' metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:48Z" name: success-200-isvc-8b307-predictor-94cb94958.18c095f8dc5844d6 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: cc625829-c7d6-49a5-9b8d-a3c9041b5e3c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 8884e7ee-3cea-4a9e-9f2b-691825c1da29 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: Scaled up replica set success-200-isvc-8b307-predictor-94cb94958 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:48Z" name: success-200-isvc-8b307-predictor.18c095f8da56e9d9 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 5cb6f661-b59b-438d-89b3-0f683c62a653 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:46:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: d2e8adef-968f-4b38-bec2-78d5c69e6085 kind: Event lastTimestamp: "2026-07-09T09:46:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:33Z" name: success-200-isvc-8b307-predictor.18c095fc58fc54e4 namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: c1b6d327-4141-406f-a97a-5fe3d53dfcfc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:46:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: d2e8adef-968f-4b38-bec2-78d5c69e6085 kind: Event lastTimestamp: "2026-07-09T09:46:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:33Z" name: success-200-isvc-8b307-predictor.18c095fc5963478b namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: bc02377a-5b97-40cc-8449-fb2c30fa61a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:46:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: d2e8adef-968f-4b38-bec2-78d5c69e6085 kind: Event lastTimestamp: "2026-07-09T09:47:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:03Z" name: success-200-isvc-8b307-predictor.18c09606d69e0940 namespace: kserve-ci-e2e-test resourceVersion: "16963" uid: 3178d972-a71a-4fcc-80c0-d0d3cf0c8cba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:46:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b307-predictor namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: d2e8adef-968f-4b38-bec2-78d5c69e6085 kind: Event lastTimestamp: "2026-07-09T09:47:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:47:03Z" name: success-200-isvc-8b307-predictor.18c09606d71c67ba namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: 247e95c5-acaa-41a9-b1b2-47ee6b18ba09 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 6a23646f-be95-443b-b850-f980ac4eba58 kind: Event lastTimestamp: "2026-07-09T09:46:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:46:37Z" name: success-200-isvc-8b307.18c095f8dab947d6 namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: ad289861-bcd6-4f18-8ff1-ed0ed790ff6e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 6a23646f-be95-443b-b850-f980ac4eba58 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: 'Failed to update status for InferenceService "success-200-isvc-8b307": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8b307": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:48Z" name: success-200-isvc-8b307.18c095f8db41d3f4 namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 5efc0598-2234-4c53-86e3-3b3904c213f1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 6a23646f-be95-443b-b850-f980ac4eba58 kind: Event lastTimestamp: "2026-07-09T09:45:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8b307": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:48Z" name: success-200-isvc-8b307.18c095f8db42179c namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: be0dfbe6-5485-48a3-ba8d-38d5b1d36ecc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b307 namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 6a23646f-be95-443b-b850-f980ac4eba58 kind: Event lastTimestamp: "2026-07-09T09:46:37Z" message: InferenceService [success-200-isvc-8b307] is Ready metadata: creationTimestamp: "2026-07-09T09:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:46:37Z" name: success-200-isvc-8b307.18c09604271b9826 namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: b6221b54-21ae-47a0-9e15-0fcbdf2f7b93 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:44:52.577137Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8d1ee-predictor-76fdc6df88-526xq to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:44:52Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ebd5e552a8 namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 43830269-dd46-4199-adad-853501cfd444 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8d1ee-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:52Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ebe06a4789 namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 228c1812-9995-4841-9d79-1501f16dc44c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:44:53Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ec134608d3 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 25f6a641-40ff-4928-84cf-eabdb2216f7f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ec14a8f26d namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: bb069d1b-7387-475c-8312-66af2d09c5c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ec17a73034 namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 36b3f496-0410-4217-b311-c2f688c9f9fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ec181307fb namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: d9175f79-c0ca-46f7-9ebe-cde51fc84665 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ec181d6845 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: cf12e8dd-9c5c-482d-a9f5-6745d5b3d08e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ec1b3374d9 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: deb27466-d648-4a71-a615-bba97c991bfa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:44:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:53Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ec1ba8c03d namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: dfa74fd6-56fc-4873-ad34-11ce4a66f3fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:44:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:45:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:31Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c095ec90bce202 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 81b2f9d6-6026-4533-abba-91c043e3e467 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:07Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c0966d00bb845e namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: 7927dba5-3248-4fe8-ad6a-6e9142807240 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: b3410a92-3fb5-47bd-885b-6831cb0b2523 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:07Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq.18c0966d00bc06ab namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 8b11518d-196f-41c8-ab9e-38daa76f941d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8d1ee-predictor-76fdc6df88 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 6e5a47fb-6cb0-45a4-815b-ba556e0069c7 kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: 'Created pod: success-200-isvc-8d1ee-predictor-76fdc6df88-526xq' metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:52Z" name: success-200-isvc-8d1ee-predictor-76fdc6df88.18c095ebd579116a namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 580890c6-3634-4d23-a7fd-431d54438b34 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 7d4539b3-22ff-47fa-83e0-c66cace70946 kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: Scaled up replica set success-200-isvc-8d1ee-predictor-76fdc6df88 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:52Z" name: success-200-isvc-8d1ee-predictor.18c095ebd2674006 namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: 39dbb031-d5a8-435c-8900-97bf1115ffc8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: b74ed17b-7bbf-4d78-b281-112666d5e3d7 kind: Event lastTimestamp: "2026-07-09T09:45:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:07Z" name: success-200-isvc-8d1ee-predictor.18c095ef50f5ed89 namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 550de71f-7432-4bfe-a55d-5ca282d9c29a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: b74ed17b-7bbf-4d78-b281-112666d5e3d7 kind: Event lastTimestamp: "2026-07-09T09:45:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:07Z" name: success-200-isvc-8d1ee-predictor.18c095ef51639cd8 namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 6b77f435-8ffc-4f6a-a2a4-75a25f5f0fe1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:45:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: b74ed17b-7bbf-4d78-b281-112666d5e3d7 kind: Event lastTimestamp: "2026-07-09T09:46:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:07Z" name: success-200-isvc-8d1ee-predictor.18c095f2d09ea11a namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: f6c3aca5-211e-4b79-95a9-16d664c9c5ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T09:45:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d1ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: b74ed17b-7bbf-4d78-b281-112666d5e3d7 kind: Event lastTimestamp: "2026-07-09T09:46:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:07Z" name: success-200-isvc-8d1ee-predictor.18c095f2d1229d4e namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 8616fd90-b946-4d6a-b3d8-7ff97e0ede8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: f7e85c27-ae71-418b-b93a-115b744df75e kind: Event lastTimestamp: "2026-07-09T09:45:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:41Z" name: success-200-isvc-8d1ee.18c095ebd27bcce6 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: f56403f7-c851-489d-9d86-e257d51b90e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: f7e85c27-ae71-418b-b93a-115b744df75e kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-8d1ee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8d1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:52Z" name: success-200-isvc-8d1ee.18c095ebd2f34b65 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: fed073d7-6576-45fa-8b42-c36c0dfbb875 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: f7e85c27-ae71-418b-b93a-115b744df75e kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8d1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:52Z" name: success-200-isvc-8d1ee.18c095ebd2f3a773 namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: afc94cdf-c602-4d33-9060-9242addfbde9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: f7e85c27-ae71-418b-b93a-115b744df75e kind: Event lastTimestamp: "2026-07-09T09:45:41Z" message: InferenceService [success-200-isvc-8d1ee] is Ready metadata: creationTimestamp: "2026-07-09T09:45:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:41Z" name: success-200-isvc-8d1ee.18c095f74d75d490 namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: 59190ff3-ae2b-4451-9519-de01d7646736 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T10:04:32.597321Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9d678-predictor-8fffb576b-dpbtd to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T10:04:32Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c096fe9494750c namespace: kserve-ci-e2e-test resourceVersion: "25244" uid: bd752374-40d6-41f8-ae33-fd4f6a3ea1b6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25255" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:04:33Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T10:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T10:04:33Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c096feaea85952 namespace: kserve-ci-e2e-test resourceVersion: "25292" uid: 448ac954-aa7c-46e1-b315-7da7193918fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:04:33Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T10:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:33Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c096feb0828a07 namespace: kserve-ci-e2e-test resourceVersion: "25305" uid: e69d456f-7204-4ae7-b2b0-93fa7329d2e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:04:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T10:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:33Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c096feb3aa8e0b namespace: kserve-ci-e2e-test resourceVersion: "25312" uid: 3b97ed1a-3f7a-4d6a-a04d-3358c9769f69 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:04:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T10:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:33Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c096feb41bdf48 namespace: kserve-ci-e2e-test resourceVersion: "25313" uid: 3d492851-986c-47ae-a1f8-8bde3ec04f69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:04:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T10:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:33Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c096feb4260ad6 namespace: kserve-ci-e2e-test resourceVersion: "25314" uid: 8c845c9a-4d94-4682-8e60-cd03fe3c7d48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:04:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T10:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:33Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c096feb91afca0 namespace: kserve-ci-e2e-test resourceVersion: "25316" uid: 57ba4d89-8eb1-4e87-adff-3a77657ef9ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:04:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:04:33Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c096feb99207f8 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: e96ff3de-f73a-4430-b3e2-593ab10e196b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:04:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:05:10Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:10Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c096fef76b3cb8 namespace: kserve-ci-e2e-test resourceVersion: "25838" uid: 5a97a4fb-c242-4ea4-86e5-83ae907d2e6f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:42Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c0970eef21bb45 namespace: kserve-ci-e2e-test resourceVersion: "26333" uid: cb9580d6-33b2-4d9a-96bc-2ba1d4e3db5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:42Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c0970eef226a37 namespace: kserve-ci-e2e-test resourceVersion: "26335" uid: b0c5cb13-d9f6-4d67-bfe0-908fd68de8a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd namespace: kserve-ci-e2e-test resourceVersion: "25234" uid: 9f775bec-d53d-4a01-a57c-8df7bc338e0b kind: Event lastTimestamp: "2026-07-09T10:05:45Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:05:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:45Z" name: success-200-isvc-9d678-predictor-8fffb576b-dpbtd.18c0970f7f80615f namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: badc1aa5-8c69-4800-ab91-69b251c824cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9d678-predictor-8fffb576b namespace: kserve-ci-e2e-test resourceVersion: "25212" uid: 433595c0-df7f-4dcc-9965-109693882676 kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: 'Created pod: success-200-isvc-9d678-predictor-8fffb576b-dpbtd' metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:04:32Z" name: success-200-isvc-9d678-predictor-8fffb576b.18c096fe941eb35d namespace: kserve-ci-e2e-test resourceVersion: "25232" uid: 18086c48-3c21-47d2-94ca-84b725e63fa6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25209" uid: 7f87630a-93d7-4822-b8e2-1e0f7df0d25e kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: Scaled up replica set success-200-isvc-9d678-predictor-8fffb576b from 0 to 1 metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:04:32Z" name: success-200-isvc-9d678-predictor.18c096fe9058adef namespace: kserve-ci-e2e-test resourceVersion: "25214" uid: 7fc0efc8-3ea9-4e51-9b07-77527951ed7d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:04:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25211" uid: 62049c67-3958-4eda-89a1-cee87f7a4876 kind: Event lastTimestamp: "2026-07-09T10:05:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:04:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:02Z" name: success-200-isvc-9d678-predictor.18c097020ed03977 namespace: kserve-ci-e2e-test resourceVersion: "25776" uid: 479fa601-c5f9-46a3-b905-63b4b0146110 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:04:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25211" uid: 62049c67-3958-4eda-89a1-cee87f7a4876 kind: Event lastTimestamp: "2026-07-09T10:05:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:04:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:02Z" name: success-200-isvc-9d678-predictor.18c097020f136d9d namespace: kserve-ci-e2e-test resourceVersion: "25777" uid: 9c3b8bc2-88ed-478c-985a-3ac1ce855914 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 62049c67-3958-4eda-89a1-cee87f7a4876 kind: Event lastTimestamp: "2026-07-09T10:05:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:32Z" name: success-200-isvc-9d678-predictor.18c097090c82e965 namespace: kserve-ci-e2e-test resourceVersion: "26162" uid: 9e4b83bb-b4db-485d-9c95-b3bf9b2084f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9d678-predictor namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 62049c67-3958-4eda-89a1-cee87f7a4876 kind: Event lastTimestamp: "2026-07-09T10:05:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:32Z" name: success-200-isvc-9d678-predictor.18c097090cd0bf4a namespace: kserve-ci-e2e-test resourceVersion: "26166" uid: ccfb2e92-2b0e-4f23-aa66-d39d44af1bef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9d678 namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: b15e7fa4-1e58-4040-871b-bcab8fa76e47 kind: Event lastTimestamp: "2026-07-09T10:05:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:20Z" name: success-200-isvc-9d678.18c096fe905db170 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: b428798d-58d1-4c49-b813-5c5b0e21bafb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9d678 namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: b15e7fa4-1e58-4040-871b-bcab8fa76e47 kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: 'Failed to update status for InferenceService "success-200-isvc-9d678": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9d678": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:04:32Z" name: success-200-isvc-9d678.18c096fe90f726a6 namespace: kserve-ci-e2e-test resourceVersion: "25217" uid: 76f8a28f-5be4-413c-b4cf-bbe62213f845 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:04:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9d678 namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: b15e7fa4-1e58-4040-871b-bcab8fa76e47 kind: Event lastTimestamp: "2026-07-09T10:04:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9d678": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:04:32Z" name: success-200-isvc-9d678.18c096fe90f791ae namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: 80dea6da-45bb-43b6-a842-a0e5c4a0613f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9d678 namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: b15e7fa4-1e58-4040-871b-bcab8fa76e47 kind: Event lastTimestamp: "2026-07-09T10:05:20Z" message: InferenceService [success-200-isvc-9d678] is Ready metadata: creationTimestamp: "2026-07-09T10:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:20Z" name: success-200-isvc-9d678.18c09709b27033fe namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: 61050c2f-8740-4b22-b3f2-72d7561672c1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:55:03.309085Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a3d13-predictor-8b899dc74-hvw4g to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:55:03Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a085b5c7f namespace: kserve-ci-e2e-test resourceVersion: "20512" uid: f253251f-4aee-427f-a6b6-35f60d15896d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a3d13-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:03Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a15db0690 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 2a82b58e-9acb-4a92-84aa-3e7e88bd3698 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:55:04Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:55:04Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a45efdc90 namespace: kserve-ci-e2e-test resourceVersion: "20574" uid: f4bccd79-ad22-488a-8e01-c9e95eefa9b3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:55:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:04Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a47423183 namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: eba35f2c-f7aa-479c-b02e-e518b61ca328 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:55:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:04Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a4a56da3b namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: dba7a7ee-bfa0-40a2-8b3a-b008973958e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:55:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:04Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a4ac6a4ec namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 599a49d8-d8b3-4d07-901a-94c40f9c9752 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:55:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:04Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a4ad0daa1 namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: 33ff1f41-7ac5-421d-9a90-0a461f5e55e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:55:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:04Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a4dff4877 namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 7c2a6a1f-5efe-47f6-8d9a-1e018a53525b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:55:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:04Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a4e8e99f9 namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: a3baa419-f88c-4d34-9874-ad66b47fae95 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:55:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:41Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0967a71dd6ef4 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: ad08b2eb-a6b2-44f5-a945-ff2b46546be3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:13Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0968a61d8660c namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 6c222a87-2329-4961-8a35-0ec94d17cf1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:56:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:13Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0968a61d8f6b3 namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: eae5d27a-3422-487c-82b7-f1bafb5935e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 5052dea4-f5e4-4694-8416-126e739b6641 kind: Event lastTimestamp: "2026-07-09T09:56:16Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:56:16Z" name: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g.18c0968afa04f809 namespace: kserve-ci-e2e-test resourceVersion: "21567" uid: 9c49195d-ad0f-4337-bc01-1fef45ef61ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a3d13-predictor-8b899dc74 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 8ebeac93-a300-4d07-a4d4-477be0f918ac kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: 'Created pod: success-200-isvc-a3d13-predictor-8b899dc74-hvw4g' metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:03Z" name: success-200-isvc-a3d13-predictor-8b899dc74.18c0967a07eee4f4 namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: 2c6325eb-591a-4575-b44d-82fab1507f80 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 2425fb0a-f360-497f-add4-2230c43b0df3 kind: Event lastTimestamp: "2026-07-09T09:55:03Z" message: Scaled up replica set success-200-isvc-a3d13-predictor-8b899dc74 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:03Z" name: success-200-isvc-a3d13-predictor.18c0967a05a00469 namespace: kserve-ci-e2e-test resourceVersion: "20486" uid: 7b7d87f4-1505-4ab9-8931-c2dd04d65245 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: e00611ea-ef82-4024-9073-f365ba67b2bf kind: Event lastTimestamp: "2026-07-09T09:55:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:48Z" name: success-200-isvc-a3d13-predictor.18c0967d83df7048 namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 3349501e-eccf-4543-ae97-cb0d4b3d9163 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: e00611ea-ef82-4024-9073-f365ba67b2bf kind: Event lastTimestamp: "2026-07-09T09:55:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:48Z" name: success-200-isvc-a3d13-predictor.18c0967d84457a0a namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: cb70fe31-a2c7-4b3d-9af4-63d2e2ad0196 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:55:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: e00611ea-ef82-4024-9073-f365ba67b2bf kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:03Z" name: success-200-isvc-a3d13-predictor.18c0968102f097b9 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 68ad5d8b-6ef0-4b1f-a0ad-54d7f6e7f037 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:55:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3d13-predictor namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: e00611ea-ef82-4024-9073-f365ba67b2bf kind: Event lastTimestamp: "2026-07-09T09:56:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:56:03Z" name: success-200-isvc-a3d13-predictor.18c09681035db8c2 namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 1c62095d-79f5-4caa-ac84-9ad44239fa79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:55:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3d13 namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 853154b7-f127-42c4-b412-b04857742a5c kind: Event lastTimestamp: "2026-07-09T09:55:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:55:51Z" name: success-200-isvc-a3d13.18c0967a05e95a41 namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 7443dad6-abca-4179-8d68-55d72f906bb1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3d13 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 853154b7-f127-42c4-b412-b04857742a5c kind: Event lastTimestamp: "2026-07-09T09:55:51Z" message: InferenceService [success-200-isvc-a3d13] is Ready metadata: creationTimestamp: "2026-07-09T09:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:55:51Z" name: success-200-isvc-a3d13.18c096852cf80c0f namespace: kserve-ci-e2e-test resourceVersion: "21179" uid: feaec42b-0a54-4a40-848c-34e3e4489181 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T10:05:42.862016Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26336" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a7fc2-predictor-864477b95b-zg9gh to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T10:05:42Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970ef0ae8c82 namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: 6e609fb8-616e-406f-9da6-f1fc01306d55 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:05:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a7fc2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:43Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970efccc63c8 namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: 798144b6-852f-40cb-b1fc-6193c2dd58d3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26346" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:05:43Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T10:05:43Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970f2e31264f namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 60a5c6f1-57f3-4ccb-bde9-6d64975849f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:05:43Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:43Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970f2fa9b594 namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: f3c20d38-3839-460f-9d71-7a154a9e72f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:05:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:43Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970f331b4dbd namespace: kserve-ci-e2e-test resourceVersion: "26418" uid: b8692194-e94e-413c-8457-38027c4b52cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:05:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:43Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970f33d0ce67 namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: 7e2de0b0-23f2-4a16-a4ff-52c15af782f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:05:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T10:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:43Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970f33da3dd5 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 652d23bd-51f0-42b5-8801-c8de73d41945 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:05:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:44Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970f36dd56ab namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 9d1bdacf-d1e2-4d0c-9e5d-e441fd53a9df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:05:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:05:44Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970f374e781a namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 576a03b5-73ee-4360-8dfa-eb35250818b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T10:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:15:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:15:00Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c0970f54b7825c namespace: kserve-ci-e2e-test resourceVersion: "30557" uid: c0fde769-1209-42e7-abec-03f5c6aae1de reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:14:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T10:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:14:57Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c097902ba94206 namespace: kserve-ci-e2e-test resourceVersion: "30521" uid: 975ac7e0-791a-4000-a126-3edabdf2501b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:14:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T10:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:14:57Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c097902ba9cfd8 namespace: kserve-ci-e2e-test resourceVersion: "30523" uid: f9b23b79-eebd-4a0c-8d38-a6e60108de23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: e1beb144-3c0f-4f83-b7d3-91dde73aaf1d kind: Event lastTimestamp: "2026-07-09T10:15:00Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T10:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:15:00Z" name: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh.18c09790c9180e1c namespace: kserve-ci-e2e-test resourceVersion: "30556" uid: 2d319f00-a179-4d17-874d-168608374378 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a7fc2-predictor-864477b95b namespace: kserve-ci-e2e-test resourceVersion: "26324" uid: ea0c5cce-70cf-4276-8e27-191e1b85c3fc kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: 'Created pod: success-200-isvc-a7fc2-predictor-864477b95b-zg9gh' metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:42Z" name: success-200-isvc-a7fc2-predictor-864477b95b.18c0970ef01a3608 namespace: kserve-ci-e2e-test resourceVersion: "26338" uid: e5b223cd-fb7b-4923-bad6-5545aa86f901 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26317" uid: a0f9b526-c995-4709-b057-622184bcef93 kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: Scaled up replica set success-200-isvc-a7fc2-predictor-864477b95b from 0 to 1 metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:05:42Z" name: success-200-isvc-a7fc2-predictor.18c0970eee4b5a7c namespace: kserve-ci-e2e-test resourceVersion: "26325" uid: 013bef96-a317-48d8-acf3-9ddc2b2e1054 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: c9a24ee3-367b-41d2-93f9-86d2beea3e9e kind: Event lastTimestamp: "2026-07-09T10:06:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:12Z" name: success-200-isvc-a7fc2-predictor.18c097126cc5f5df namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: c23a3090-94fb-43f5-87ea-783c01ad1645 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:05:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: c9a24ee3-367b-41d2-93f9-86d2beea3e9e kind: Event lastTimestamp: "2026-07-09T10:06:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:12Z" name: success-200-isvc-a7fc2-predictor.18c097126d2ab594 namespace: kserve-ci-e2e-test resourceVersion: "26676" uid: 078d6e4a-6738-47b7-81d2-7b307fdf8814 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:06:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: c9a24ee3-367b-41d2-93f9-86d2beea3e9e kind: Event lastTimestamp: "2026-07-09T10:06:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:42Z" name: success-200-isvc-a7fc2-predictor.18c097196b4388a9 namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: fafef251-afef-4dda-9a48-63620cdd492f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:06:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a7fc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: c9a24ee3-367b-41d2-93f9-86d2beea3e9e kind: Event lastTimestamp: "2026-07-09T10:06:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T10:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:42Z" name: success-200-isvc-a7fc2-predictor.18c097196baadd3f namespace: kserve-ci-e2e-test resourceVersion: "26990" uid: 93cc4249-f9b7-482c-a29a-c379b578773f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: e34bfb0b-12c3-4e3b-8893-906dd57f8295 kind: Event lastTimestamp: "2026-07-09T10:06:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:30Z" name: success-200-isvc-a7fc2.18c0970eee53eb01 namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 67c11116-3cb6-4f98-8e48-85a968195dc8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: e34bfb0b-12c3-4e3b-8893-906dd57f8295 kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: 'Failed to update status for InferenceService "success-200-isvc-a7fc2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a7fc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:42Z" name: success-200-isvc-a7fc2.18c0970eef1dfc47 namespace: kserve-ci-e2e-test resourceVersion: "26331" uid: 3d5c4c4d-1628-4580-8fba-02d0e0984381 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:05:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: e34bfb0b-12c3-4e3b-8893-906dd57f8295 kind: Event lastTimestamp: "2026-07-09T10:05:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a7fc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T10:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:05:42Z" name: success-200-isvc-a7fc2.18c0970eef1e486a namespace: kserve-ci-e2e-test resourceVersion: "26334" uid: eee81b3c-9547-4c0c-989a-51b9c3ed02e4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a7fc2 namespace: kserve-ci-e2e-test resourceVersion: "26881" uid: e34bfb0b-12c3-4e3b-8893-906dd57f8295 kind: Event lastTimestamp: "2026-07-09T10:06:30Z" message: InferenceService [success-200-isvc-a7fc2] is Ready metadata: creationTimestamp: "2026-07-09T10:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:30Z" name: success-200-isvc-a7fc2.18c0971a101aed89 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 466cfe20-b408-4679-bb85-8c1cc9b4c284 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:54:07.408319Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b04db-predictor-869d74887d-c7bcl to ip-10-0-143-234.ec2.internal metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:54:07Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d046949d4 namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 22c6ebc4-8ede-45c8-b03a-d32d94b5801e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:54:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b04db-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:08Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d0f46aad3 namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 72f83c2e-db59-4e27-a07b-03c955a99137 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19945" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:54:09Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:54:09Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d77ce5e15 namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: c9efd871-6eb5-421e-8ea1-4188415f1d54 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:54:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1684" already present on machine metadata: creationTimestamp: "2026-07-09T09:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:09Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d793c4b53 namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: abc5d915-fa6b-4c4f-99b4-442d635f2f76 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:54:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T09:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:09Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d7c6b82c4 namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: 5c0a06d4-8984-41ec-81d7-ff2a63385f82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:54:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T09:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:09Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d7cdf8cce namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: a2c5c401-e95e-421f-bc23-b0e608f9cfc6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:54:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T09:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:09Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d7ceb9e50 namespace: kserve-ci-e2e-test resourceVersion: "20009" uid: 00c5d10d-0a60-4859-9e4e-2b273d20d6d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:54:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T09:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:09Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d7f67e510 namespace: kserve-ci-e2e-test resourceVersion: "20010" uid: dc2e7d93-875f-481c-aa3a-2b3c4d0f866f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:54:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:09Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d7fd96462 namespace: kserve-ci-e2e-test resourceVersion: "20011" uid: b2aa9b9e-998f-4d0a-bb86-999bf98837a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:54:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:54:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T09:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:45Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0966d962f19b8 namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 7b0e6045-bc09-4e63-b92c-5ad66d6151e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:17Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0967d5dc9ae63 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 8f461b9c-f173-49b9-a402-fcb382687e57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b04db-predictor-869d74887d-c7bcl namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: ee03df07-0340-4983-b3e2-b5e90c60705e kind: Event lastTimestamp: "2026-07-09T09:55:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T09:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:55:17Z" name: success-200-isvc-b04db-predictor-869d74887d-c7bcl.18c0967d5dca67db namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: fa20ce75-f9a6-4029-a665-5b151256e1fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-234.ec2.internal source: component: kubelet host: ip-10-0-143-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b04db-predictor-869d74887d namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: bde6886e-4fe7-4d47-924c-9b88bcee0f96 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: 'Created pod: success-200-isvc-b04db-predictor-869d74887d-c7bcl' metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:07Z" name: success-200-isvc-b04db-predictor-869d74887d.18c0966d03ecea8b namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 4f4593d2-3410-48be-8b35-2de5c3aca803 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 85ba783d-bcc0-4215-a50b-6c98eefa334a kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: Scaled up replica set success-200-isvc-b04db-predictor-869d74887d from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:07Z" name: success-200-isvc-b04db-predictor.18c0966d006379a3 namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 9f021b2b-5a42-4d23-88f6-586f643b2be0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:54:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "19901" uid: a0677848-6d32-4224-9c37-b8b70b82b517 kind: Event lastTimestamp: "2026-07-09T09:54:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:52Z" name: success-200-isvc-b04db-predictor.18c096707ecb7805 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: 6fd2b5bb-c605-44e7-a6a6-43b144b31eb3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T09:54:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "19901" uid: a0677848-6d32-4224-9c37-b8b70b82b517 kind: Event lastTimestamp: "2026-07-09T09:54:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T09:54:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:54:52Z" name: success-200-isvc-b04db-predictor.18c096707f3514b9 namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: d30f283c-7e68-46f8-b4d2-9f9460087217 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: a0677848-6d32-4224-9c37-b8b70b82b517 kind: Event lastTimestamp: "2026-07-09T09:55:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:07Z" name: success-200-isvc-b04db-predictor.18c0967afb290f93 namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 780fb417-e678-49e4-9f8c-fe76580a4d5b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:55:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b04db-predictor namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: a0677848-6d32-4224-9c37-b8b70b82b517 kind: Event lastTimestamp: "2026-07-09T09:55:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:55:07Z" name: success-200-isvc-b04db-predictor.18c0967afb717e50 namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: 80769955-a698-4820-a209-945982348520 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b04db namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 42c1ccfa-6a4b-492c-90c4-dbfb6e643f39 kind: Event lastTimestamp: "2026-07-09T09:54:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:55Z" name: success-200-isvc-b04db.18c0966d0094291b namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 2c3978a2-1c90-4809-b871-3ae7d5a71a09 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b04db namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 42c1ccfa-6a4b-492c-90c4-dbfb6e643f39 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: 'Failed to update status for InferenceService "success-200-isvc-b04db": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b04db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:07Z" name: success-200-isvc-b04db.18c0966d011c6fa6 namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: 0b712ceb-bad7-48a7-816e-96bc93d9032b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b04db namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 42c1ccfa-6a4b-492c-90c4-dbfb6e643f39 kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b04db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:07Z" name: success-200-isvc-b04db.18c0966d011ccf45 namespace: kserve-ci-e2e-test resourceVersion: "19913" uid: b78af79e-6c88-4bc9-9b06-db24a65914c6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b04db namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: 42c1ccfa-6a4b-492c-90c4-dbfb6e643f39 kind: Event lastTimestamp: "2026-07-09T09:54:55Z" message: InferenceService [success-200-isvc-b04db] is Ready metadata: creationTimestamp: "2026-07-09T09:54:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:54:55Z" name: success-200-isvc-b04db.18c09678515ff71d namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: 89e4160b-efa9-4604-bd76-2599ce6ecefb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:44:38.241680Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1980c-84fdbd9b6f-d79c9 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: ec8a48ba-77aa-4691-9b6c-e0b99829569a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1980c-84fdbd9b6f-d79c9 to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:44:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:44:38Z" name: switch-graph-1980c-84fdbd9b6f-d79c9.18c095e87f6f9767 namespace: kserve-ci-e2e-test resourceVersion: "15315" uid: 8fcb27be-9f12-42f0-ab2c-41d29224200e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1980c-84fdbd9b6f-d79c9 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: ec8a48ba-77aa-4691-9b6c-e0b99829569a kind: Event lastTimestamp: "2026-07-09T09:44:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-1980c-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T09:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:38Z" name: switch-graph-1980c-84fdbd9b6f-d79c9.18c095e890c21736 namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: c8e7f473-2092-434b-afd8-b200e0adbca1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:39Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1980c-84fdbd9b6f-d79c9 namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: ec8a48ba-77aa-4691-9b6c-e0b99829569a kind: Event lastTimestamp: "2026-07-09T09:44:39Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:44:39Z" name: switch-graph-1980c-84fdbd9b6f-d79c9.18c095e8bc2337a3 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1a6b0d8c-c133-4640-903c-dfc275d14e06 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1980c} kind: Pod name: switch-graph-1980c-84fdbd9b6f-d79c9 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: ec8a48ba-77aa-4691-9b6c-e0b99829569a kind: Event lastTimestamp: "2026-07-09T09:44:39Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" metadata: creationTimestamp: "2026-07-09T09:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:39Z" name: switch-graph-1980c-84fdbd9b6f-d79c9.18c095e8bd91e687 namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 72b0a716-5d2c-45eb-b93c-d6ee638aceee reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1980c} kind: Pod name: switch-graph-1980c-84fdbd9b6f-d79c9 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: ec8a48ba-77aa-4691-9b6c-e0b99829569a kind: Event lastTimestamp: "2026-07-09T09:44:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" in 2.278s (2.278s including waiting). Image size: 218072676 bytes.' metadata: creationTimestamp: "2026-07-09T09:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:41Z" name: switch-graph-1980c-84fdbd9b6f-d79c9.18c095e945642e58 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: 18cd2ad0-5f29-4b99-b769-0d961ce28467 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1980c} kind: Pod name: switch-graph-1980c-84fdbd9b6f-d79c9 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: ec8a48ba-77aa-4691-9b6c-e0b99829569a kind: Event lastTimestamp: "2026-07-09T09:44:41Z" message: 'Created container: switch-graph-1980c' metadata: creationTimestamp: "2026-07-09T09:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:41Z" name: switch-graph-1980c-84fdbd9b6f-d79c9.18c095e9483cb408 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: b85f59c4-c7f3-4dd6-b35e-78d2679eeccb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1980c} kind: Pod name: switch-graph-1980c-84fdbd9b6f-d79c9 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: ec8a48ba-77aa-4691-9b6c-e0b99829569a kind: Event lastTimestamp: "2026-07-09T09:44:41Z" message: Started container switch-graph-1980c metadata: creationTimestamp: "2026-07-09T09:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:41Z" name: switch-graph-1980c-84fdbd9b6f-d79c9.18c095e9489826c5 namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: 212da26b-840e-49b0-ae5d-1896c631cf9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1980c} kind: Pod name: switch-graph-1980c-84fdbd9b6f-d79c9 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: ec8a48ba-77aa-4691-9b6c-e0b99829569a kind: Event lastTimestamp: "2026-07-09T09:44:52Z" message: Stopping container switch-graph-1980c metadata: creationTimestamp: "2026-07-09T09:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:44:52Z" name: switch-graph-1980c-84fdbd9b6f-d79c9.18c095ebc9acc79b namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: 084cb218-abcc-4a66-8601-09f0f925b024 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1980c} kind: Pod name: switch-graph-1980c-84fdbd9b6f-d79c9 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: ec8a48ba-77aa-4691-9b6c-e0b99829569a kind: Event lastTimestamp: "2026-07-09T09:45:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T09:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:18Z" name: switch-graph-1980c-84fdbd9b6f-d79c9.18c095ec0d537a88 namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: 4fb8ec4f-21d3-421d-93a8-da5947680958 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1980c-84fdbd9b6f namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 8601e48d-013f-459a-96b4-3003e149ad0c kind: Event lastTimestamp: "2026-07-09T09:44:38Z" message: 'Created pod: switch-graph-1980c-84fdbd9b6f-d79c9' metadata: creationTimestamp: "2026-07-09T09:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:38Z" name: switch-graph-1980c-84fdbd9b6f.18c095e87ef9b650 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: c030b862-1449-414c-8f70-6c6d9463d3b8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1980c namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: 52cc2b0f-d672-455a-b522-8e30d72486cc kind: Event lastTimestamp: "2026-07-09T09:44:38Z" message: Scaled up replica set switch-graph-1980c-84fdbd9b6f from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:44:38Z" name: switch-graph-1980c.18c095e87e040f37 namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: a3be0c43-a7f7-4d5d-853f-de0621ee8e50 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1980c namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: c2b962f2-a2c3-4ef1-b44d-a4d2f3fb96e7 kind: Event lastTimestamp: "2026-07-09T09:44:38Z" message: InferenceGraph [switch-graph-1980c] is Ready metadata: creationTimestamp: "2026-07-09T09:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:38Z" name: switch-graph-1980c.18c095e881f8c344 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 6727c2f8-c40b-4ab6-b78a-d06c281bbfe3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1980c namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: c2b962f2-a2c3-4ef1-b44d-a4d2f3fb96e7 kind: Event lastTimestamp: "2026-07-09T09:44:38Z" message: 'Failed to update status for InferenceGraph "switch-graph-1980c": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-1980c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:38Z" name: switch-graph-1980c.18c095e88459b645 namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 9436817a-6686-4976-9fac-326dab8d9671 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1980c namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: c2b962f2-a2c3-4ef1-b44d-a4d2f3fb96e7 kind: Event lastTimestamp: "2026-07-09T09:44:38Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-1980c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:44:38Z" name: switch-graph-1980c.18c095e88459fa1e namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 498f1361-d4ea-4080-8731-805c30e7d767 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T10:06:28.631452Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-52f9b-557dd99dc5-s2n4c namespace: kserve-ci-e2e-test resourceVersion: "26836" uid: 2ce8c242-90c3-4b50-8ebe-e51dbdf7a478 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-52f9b-557dd99dc5-s2n4c to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T10:06:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T10:06:28Z" name: switch-graph-52f9b-557dd99dc5-s2n4c.18c0971998c0c492 namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: 8f786de4-b533-4c03-be03-920f09e32e07 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:29Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-52f9b-557dd99dc5-s2n4c namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: 2ce8c242-90c3-4b50-8ebe-e51dbdf7a478 kind: Event lastTimestamp: "2026-07-09T10:06:29Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T10:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T10:06:29Z" name: switch-graph-52f9b-557dd99dc5-s2n4c.18c09719b23e3111 namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 0d606ee4-3249-45aa-ae94-9541fefb0857 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-52f9b} kind: Pod name: switch-graph-52f9b-557dd99dc5-s2n4c namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 2ce8c242-90c3-4b50-8ebe-e51dbdf7a478 kind: Event lastTimestamp: "2026-07-09T10:06:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T10:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:29Z" name: switch-graph-52f9b-557dd99dc5-s2n4c.18c09719b3a36a86 namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: af9ef2f4-e3af-463a-b2e3-5b975231fceb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-52f9b} kind: Pod name: switch-graph-52f9b-557dd99dc5-s2n4c namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 2ce8c242-90c3-4b50-8ebe-e51dbdf7a478 kind: Event lastTimestamp: "2026-07-09T10:06:29Z" message: 'Created container: switch-graph-52f9b' metadata: creationTimestamp: "2026-07-09T10:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:29Z" name: switch-graph-52f9b-557dd99dc5-s2n4c.18c09719b63b1cad namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: e0c7f735-baed-4c7f-8984-fa08760f941c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-52f9b} kind: Pod name: switch-graph-52f9b-557dd99dc5-s2n4c namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 2ce8c242-90c3-4b50-8ebe-e51dbdf7a478 kind: Event lastTimestamp: "2026-07-09T10:06:29Z" message: Started container switch-graph-52f9b metadata: creationTimestamp: "2026-07-09T10:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:06:29Z" name: switch-graph-52f9b-557dd99dc5-s2n4c.18c09719b6a6e281 namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: 06be6714-37ba-4374-9a81-124bd697c410 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:22:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-52f9b} kind: Pod name: switch-graph-52f9b-557dd99dc5-s2n4c namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 2ce8c242-90c3-4b50-8ebe-e51dbdf7a478 kind: Event lastTimestamp: "2026-07-09T10:22:47Z" message: Stopping container switch-graph-52f9b metadata: creationTimestamp: "2026-07-09T10:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:22:47Z" name: switch-graph-52f9b-557dd99dc5-s2n4c.18c097fd9a7ba779 namespace: kserve-ci-e2e-test resourceVersion: "33763" uid: 75965a07-fecf-4493-9e41-b5236bfaf76f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T10:22:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-52f9b} kind: Pod name: switch-graph-52f9b-557dd99dc5-s2n4c namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 2ce8c242-90c3-4b50-8ebe-e51dbdf7a478 kind: Event lastTimestamp: "2026-07-09T10:23:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T10:22:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T10:23:10Z" name: switch-graph-52f9b-557dd99dc5-s2n4c.18c097fe2f29bc19 namespace: kserve-ci-e2e-test resourceVersion: "33998" uid: 9aa77bbe-80a6-4bb7-8a87-c27a46d576d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-52f9b-557dd99dc5 namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: a0007cfc-971e-4deb-8237-516193205374 kind: Event lastTimestamp: "2026-07-09T10:06:28Z" message: 'Created pod: switch-graph-52f9b-557dd99dc5-s2n4c' metadata: creationTimestamp: "2026-07-09T10:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:28Z" name: switch-graph-52f9b-557dd99dc5.18c0971998531288 namespace: kserve-ci-e2e-test resourceVersion: "26838" uid: c7b4f4fb-b058-49c3-9933-f277f3edea22 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26829" uid: eb63b971-2c0f-4210-9a28-a6d5390b2439 kind: Event lastTimestamp: "2026-07-09T10:06:28Z" message: Scaled up replica set switch-graph-52f9b-557dd99dc5 from 0 to 1 metadata: creationTimestamp: "2026-07-09T10:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:28Z" name: switch-graph-52f9b.18c09719973a6b65 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 698e8926-111a-414f-bba5-17866ecf111a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T10:06:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 8c0c87f0-1fb9-441f-b8c2-3f9020f5acd9 kind: Event lastTimestamp: "2026-07-09T10:06:28Z" message: InferenceGraph [switch-graph-52f9b] is Ready metadata: creationTimestamp: "2026-07-09T10:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T10:06:28Z" name: switch-graph-52f9b.18c097199b466650 namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: 48ff0e6f-bb2b-4304-86c0-516822debfe6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:06:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 431330ec-77f9-4da5-8ad0-1a432f9f04e6 kind: Event lastTimestamp: "2026-07-09T10:06:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:58Z" name: switch-graph-52f9b.18c0971d15a64407 namespace: kserve-ci-e2e-test resourceVersion: "27151" uid: f0eef045-7766-4230-8c0f-deb7c3bc01e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T10:06:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-52f9b namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 431330ec-77f9-4da5-8ad0-1a432f9f04e6 kind: Event lastTimestamp: "2026-07-09T10:06:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T10:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T10:06:58Z" name: switch-graph-52f9b.18c0971d16115c76 namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 9f763ab7-e704-495a-ab82-b8eb342473f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T09:45:52.609952Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8d1ee-7c6544c954-fv9qq namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: a655b312-6a82-4ba3-8a8a-9465580aab2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8d1ee-7c6544c954-fv9qq to ip-10-0-141-211.ec2.internal metadata: creationTimestamp: "2026-07-09T09:45:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T09:45:52Z" name: switch-graph-8d1ee-7c6544c954-fv9qq.18c095f9d0215791 namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: b75b5710-be53-417b-ab63-eb9aa1b8ef29 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-748cc78cb6-t825m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:53Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8d1ee-7c6544c954-fv9qq namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: a655b312-6a82-4ba3-8a8a-9465580aab2f kind: Event lastTimestamp: "2026-07-09T09:45:53Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T09:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T09:45:53Z" name: switch-graph-8d1ee-7c6544c954-fv9qq.18c095f9e9737e8f namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: 58bb9191-500e-4bf4-a3dc-b1b9fec87c5e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d1ee} kind: Pod name: switch-graph-8d1ee-7c6544c954-fv9qq namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: a655b312-6a82-4ba3-8a8a-9465580aab2f kind: Event lastTimestamp: "2026-07-09T09:45:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:264e4696bc4acb7c1e00cabab43621f26f8fb70223f71f60afed1f1b1add26b9" already present on machine metadata: creationTimestamp: "2026-07-09T09:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:53Z" name: switch-graph-8d1ee-7c6544c954-fv9qq.18c095f9ead0688d namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 786469a5-1260-497c-8ce7-ccff14e4e89d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d1ee} kind: Pod name: switch-graph-8d1ee-7c6544c954-fv9qq namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: a655b312-6a82-4ba3-8a8a-9465580aab2f kind: Event lastTimestamp: "2026-07-09T09:45:53Z" message: 'Created container: switch-graph-8d1ee' metadata: creationTimestamp: "2026-07-09T09:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:53Z" name: switch-graph-8d1ee-7c6544c954-fv9qq.18c095f9edaa82ab namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 4b5a9b4d-1022-429f-ac67-b01599124778 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d1ee} kind: Pod name: switch-graph-8d1ee-7c6544c954-fv9qq namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: a655b312-6a82-4ba3-8a8a-9465580aab2f kind: Event lastTimestamp: "2026-07-09T09:45:53Z" message: Started container switch-graph-8d1ee metadata: creationTimestamp: "2026-07-09T09:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:45:53Z" name: switch-graph-8d1ee-7c6544c954-fv9qq.18c095f9ee17625a namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: 93aa0596-39f0-4a01-b2ba-2ac94fa0b4b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d1ee} kind: Pod name: switch-graph-8d1ee-7c6544c954-fv9qq namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: a655b312-6a82-4ba3-8a8a-9465580aab2f kind: Event lastTimestamp: "2026-07-09T09:54:07Z" message: Stopping container switch-graph-8d1ee metadata: creationTimestamp: "2026-07-09T09:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:07Z" name: switch-graph-8d1ee-7c6544c954-fv9qq.18c0966cf828455f namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: f24f17dd-1ae0-416d-8a51-43e1eaaee670 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T09:54:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d1ee} kind: Pod name: switch-graph-8d1ee-7c6544c954-fv9qq namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: a655b312-6a82-4ba3-8a8a-9465580aab2f kind: Event lastTimestamp: "2026-07-09T09:54:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T09:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T09:54:34Z" name: switch-graph-8d1ee-7c6544c954-fv9qq.18c0966d94a3322c namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: c34b583a-36ab-4573-aba7-fd4c19233447 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-211.ec2.internal source: component: kubelet host: ip-10-0-141-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8d1ee-7c6544c954 namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: 3586bd0f-e9df-4127-841f-9db05c395dd5 kind: Event lastTimestamp: "2026-07-09T09:45:52Z" message: 'Created pod: switch-graph-8d1ee-7c6544c954-fv9qq' metadata: creationTimestamp: "2026-07-09T09:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:52Z" name: switch-graph-8d1ee-7c6544c954.18c095f9cfb23730 namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: ede0d8b2-ccf2-4f2f-987a-8b3278b29b35 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 23dec814-0d9d-4baa-b83c-786833b965ba kind: Event lastTimestamp: "2026-07-09T09:45:52Z" message: Scaled up replica set switch-graph-8d1ee-7c6544c954 from 0 to 1 metadata: creationTimestamp: "2026-07-09T09:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:45:52Z" name: switch-graph-8d1ee.18c095f9cef0cecd namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 959d0da6-2c8b-4144-bd26-b3a7a882e4b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: def12fb4-77fc-4312-8e53-d2c55637f5e3 kind: Event lastTimestamp: "2026-07-09T09:45:52Z" message: InferenceGraph [switch-graph-8d1ee] is Ready metadata: creationTimestamp: "2026-07-09T09:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:52Z" name: switch-graph-8d1ee.18c095f9d48064df namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: 922063f6-5479-41e8-935c-e6d26c0da314 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: def12fb4-77fc-4312-8e53-d2c55637f5e3 kind: Event lastTimestamp: "2026-07-09T09:45:52Z" message: 'Failed to update status for InferenceGraph "switch-graph-8d1ee": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-8d1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:52Z" name: switch-graph-8d1ee.18c095f9d6bcc7c0 namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: 63d54ed1-5ed9-4915-8963-d1ba2f3f465f reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:45:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: def12fb4-77fc-4312-8e53-d2c55637f5e3 kind: Event lastTimestamp: "2026-07-09T09:45:52Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-8d1ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T09:45:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T09:45:52Z" name: switch-graph-8d1ee.18c095f9d6bd0ce6 namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 8ceda5e2-3e75-41f8-9258-3b80f67563cf reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 60288c11-ead6-4597-9876-4fa61f6d0ef5 kind: Event lastTimestamp: "2026-07-09T09:46:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:07Z" name: switch-graph-8d1ee.18c095fd4d48e65b namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 23a8dae1-358f-4af6-ab14-2584aeaa8b0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T09:46:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8d1ee namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 60288c11-ead6-4597-9876-4fa61f6d0ef5 kind: Event lastTimestamp: "2026-07-09T09:46:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T09:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T09:46:07Z" name: switch-graph-8d1ee.18c095fd4d9fb83c namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 61a752cd-6a9f-4d7c-9579-3b560dcaf68d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34025"