--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:17.125980Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4a23a-84545c9fbf-4dlff namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 4615d28e-abc9-4ded-b26e-c955cb9e9c6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-4a23a-84545c9fbf-4dlff to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:17Z" name: ensemble-graph-4a23a-84545c9fbf-4dlff.18c684823868ce1f namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: c3b4eeb2-0b1d-4429-9638-c95b8918724b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4a23a-84545c9fbf-4dlff namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 4615d28e-abc9-4ded-b26e-c955cb9e9c6c kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-4a23a-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:17Z" name: ensemble-graph-4a23a-84545c9fbf-4dlff.18c684824476680f namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: b7f537a4-f3a3-4df9-93cf-220c5dd9db7c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:18Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4a23a-84545c9fbf-4dlff namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 4615d28e-abc9-4ded-b26e-c955cb9e9c6c kind: Event lastTimestamp: "2026-07-28T17:33:18Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:33:18Z" name: ensemble-graph-4a23a-84545c9fbf-4dlff.18c68482759aa0db namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 2d3e1a48-6b3e-4636-aa58-438335c694c3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4a23a} kind: Pod name: ensemble-graph-4a23a-84545c9fbf-4dlff namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 4615d28e-abc9-4ded-b26e-c955cb9e9c6c kind: Event lastTimestamp: "2026-07-28T17:33:18Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" metadata: creationTimestamp: "2026-07-28T17:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:18Z" name: ensemble-graph-4a23a-84545c9fbf-4dlff.18c6848276eae071 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: e625ee17-20dd-4f63-8d09-95e7a2760bd3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4a23a} kind: Pod name: ensemble-graph-4a23a-84545c9fbf-4dlff namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 4615d28e-abc9-4ded-b26e-c955cb9e9c6c kind: Event lastTimestamp: "2026-07-28T17:33:20Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" in 2.378s (2.378s including waiting). Image size: 218226779 bytes.' metadata: creationTimestamp: "2026-07-28T17:33:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:20Z" name: ensemble-graph-4a23a-84545c9fbf-4dlff.18c6848304adbb5b namespace: kserve-ci-e2e-test resourceVersion: "21579" uid: 507f7d0a-4aca-4717-bc01-f7ece5bf9ce5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4a23a} kind: Pod name: ensemble-graph-4a23a-84545c9fbf-4dlff namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 4615d28e-abc9-4ded-b26e-c955cb9e9c6c kind: Event lastTimestamp: "2026-07-28T17:33:20Z" message: 'Created container: ensemble-graph-4a23a' metadata: creationTimestamp: "2026-07-28T17:33:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:20Z" name: ensemble-graph-4a23a-84545c9fbf-4dlff.18c6848307b1b023 namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: 2dc232ad-64f7-4ace-8f9c-bfdbe8364e44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4a23a} kind: Pod name: ensemble-graph-4a23a-84545c9fbf-4dlff namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 4615d28e-abc9-4ded-b26e-c955cb9e9c6c kind: Event lastTimestamp: "2026-07-28T17:33:20Z" message: Started container ensemble-graph-4a23a metadata: creationTimestamp: "2026-07-28T17:33:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:20Z" name: ensemble-graph-4a23a-84545c9fbf-4dlff.18c6848308256b15 namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: cdb13c80-f106-4ba5-8cbc-4c1b79eae138 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4a23a} kind: Pod name: ensemble-graph-4a23a-84545c9fbf-4dlff namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 4615d28e-abc9-4ded-b26e-c955cb9e9c6c kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Stopping container ensemble-graph-4a23a metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:31Z" name: ensemble-graph-4a23a-84545c9fbf-4dlff.18c684857fb1d905 namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 2af21322-055e-406f-9eac-b7375c8843a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4a23a} kind: Pod name: ensemble-graph-4a23a-84545c9fbf-4dlff namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 4615d28e-abc9-4ded-b26e-c955cb9e9c6c kind: Event lastTimestamp: "2026-07-28T17:33:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:57Z" name: ensemble-graph-4a23a-84545c9fbf-4dlff.18c68485c237d4ad namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: fbf410c0-dbd8-4e19-b0a3-72c76044a30b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-4a23a-84545c9fbf namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 10d94432-0581-4444-a1cf-d49010aaf56f kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'Created pod: ensemble-graph-4a23a-84545c9fbf-4dlff' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:17Z" name: ensemble-graph-4a23a-84545c9fbf.18c6848237d159d8 namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 156f9a11-2873-4c02-9b16-0b9546eb4c07 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-4a23a namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 4f6c289a-7049-44f1-8246-aaa018d639d9 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: Scaled up replica set ensemble-graph-4a23a-84545c9fbf from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:17Z" name: ensemble-graph-4a23a.18c6848237014054 namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 5cbc0e06-2994-49fc-8681-cb093cf845f9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4a23a namespace: kserve-ci-e2e-test resourceVersion: "21503" uid: 5dd4fbb1-fa95-4c48-af9e-41b3ea5ca41c kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: InferenceGraph [ensemble-graph-4a23a] is Ready metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:17Z" name: ensemble-graph-4a23a.18c684823d29c78b namespace: kserve-ci-e2e-test resourceVersion: "21505" uid: 1dce160b-6220-43f6-9743-97b43875325a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4a23a namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 5dd4fbb1-fa95-4c48-af9e-41b3ea5ca41c kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-4a23a": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-4a23a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:17Z" name: ensemble-graph-4a23a.18c684823fc017ad namespace: kserve-ci-e2e-test resourceVersion: "21507" uid: 02287478-6efb-49b0-a707-49f124b86141 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4a23a namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 5dd4fbb1-fa95-4c48-af9e-41b3ea5ca41c kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-4a23a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:17Z" name: ensemble-graph-4a23a.18c684823fc07ea7 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: d3dbe47f-6853-4c61-914b-1abe1588e5bc reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:34:31.454706Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d1fc0-7565fb94f7-tkz2b namespace: kserve-ci-e2e-test resourceVersion: "22538" uid: ba8f91a1-631d-493e-ad7c-91ef20a9d474 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-d1fc0-7565fb94f7-tkz2b to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:34:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:34:31Z" name: ensemble-graph-d1fc0-7565fb94f7-tkz2b.18c6849386bf1c5e namespace: kserve-ci-e2e-test resourceVersion: "22545" uid: 89677c59-7272-4aff-972a-b8db52d58fa5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:31Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d1fc0-7565fb94f7-tkz2b namespace: kserve-ci-e2e-test resourceVersion: "22551" uid: ba8f91a1-631d-493e-ad7c-91ef20a9d474 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:34:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:34:31Z" name: ensemble-graph-d1fc0-7565fb94f7-tkz2b.18c684939fff9d34 namespace: kserve-ci-e2e-test resourceVersion: "22563" uid: 51c3f601-7110-4c41-a174-e813e11dee61 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1fc0} kind: Pod name: ensemble-graph-d1fc0-7565fb94f7-tkz2b namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: ba8f91a1-631d-493e-ad7c-91ef20a9d474 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:31Z" name: ensemble-graph-d1fc0-7565fb94f7-tkz2b.18c68493a1323729 namespace: kserve-ci-e2e-test resourceVersion: "22565" uid: 3e7a76ed-d947-4a7c-8f5e-fbda80e08125 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1fc0} kind: Pod name: ensemble-graph-d1fc0-7565fb94f7-tkz2b namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: ba8f91a1-631d-493e-ad7c-91ef20a9d474 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: 'Created container: ensemble-graph-d1fc0' metadata: creationTimestamp: "2026-07-28T17:34:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:31Z" name: ensemble-graph-d1fc0-7565fb94f7-tkz2b.18c68493a40f0b11 namespace: kserve-ci-e2e-test resourceVersion: "22568" uid: 8d444225-9f77-4eee-90ee-e750dc8e4f66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1fc0} kind: Pod name: ensemble-graph-d1fc0-7565fb94f7-tkz2b namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: ba8f91a1-631d-493e-ad7c-91ef20a9d474 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: Started container ensemble-graph-d1fc0 metadata: creationTimestamp: "2026-07-28T17:34:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:31Z" name: ensemble-graph-d1fc0-7565fb94f7-tkz2b.18c68493a47810b5 namespace: kserve-ci-e2e-test resourceVersion: "22569" uid: 23d481c6-25ed-456b-a21e-fa6150fbb65c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1fc0} kind: Pod name: ensemble-graph-d1fc0-7565fb94f7-tkz2b namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: ba8f91a1-631d-493e-ad7c-91ef20a9d474 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Stopping container ensemble-graph-d1fc0 metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: ensemble-graph-d1fc0-7565fb94f7-tkz2b.18c68506af3bef92 namespace: kserve-ci-e2e-test resourceVersion: "26383" uid: dc080ba7-77dc-455e-8a93-1bfdc03ac2db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:42:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1fc0} kind: Pod name: ensemble-graph-d1fc0-7565fb94f7-tkz2b namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: ba8f91a1-631d-493e-ad7c-91ef20a9d474 kind: Event lastTimestamp: "2026-07-28T17:43:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:13Z" name: ensemble-graph-d1fc0-7565fb94f7-tkz2b.18c685074731cddc namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 12453dd0-e201-4634-a0a1-5c0bdf9ed6fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-d1fc0-7565fb94f7 namespace: kserve-ci-e2e-test resourceVersion: "22535" uid: 2603dc33-cb55-492e-98d6-44588f7b6473 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: 'Created pod: ensemble-graph-d1fc0-7565fb94f7-tkz2b' metadata: creationTimestamp: "2026-07-28T17:34:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:31Z" name: ensemble-graph-d1fc0-7565fb94f7.18c68493864ca6c4 namespace: kserve-ci-e2e-test resourceVersion: "22540" uid: 7e47b96c-c566-44ad-b3b4-c8c5fd0bff1c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: 93e681be-a275-4edc-9776-eeda476ce1c8 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: Scaled up replica set ensemble-graph-d1fc0-7565fb94f7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:34:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:31Z" name: ensemble-graph-d1fc0.18c6849385936104 namespace: kserve-ci-e2e-test resourceVersion: "22536" uid: 01845eff-bacd-438c-9c79-90a565ec6670 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "22557" uid: ef27570b-d5bd-4f41-95d2-2f4eea6cb763 kind: Event lastTimestamp: "2026-07-28T17:34:31Z" message: InferenceGraph [ensemble-graph-d1fc0] is Ready metadata: creationTimestamp: "2026-07-28T17:34:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:31Z" name: ensemble-graph-d1fc0.18c684938b6d9ca6 namespace: kserve-ci-e2e-test resourceVersion: "22560" uid: 7377b173-92ba-4bd4-b094-8112d970cc38 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "22534" uid: 88d8ac58-985e-4a67-9e7c-6be89fe02150 kind: Event lastTimestamp: "2026-07-28T17:35:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:16Z" name: ensemble-graph-d1fc0.18c68497047eed1e namespace: kserve-ci-e2e-test resourceVersion: "22945" uid: 4a7921f8-d1c9-4e6a-8e4e-3d4a25a3d88d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "22534" uid: 88d8ac58-985e-4a67-9e7c-6be89fe02150 kind: Event lastTimestamp: "2026-07-28T17:35:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:16Z" name: ensemble-graph-d1fc0.18c6849704d4dfaa namespace: kserve-ci-e2e-test resourceVersion: "22946" uid: 9aab2941-7201-40d2-990b-bf276d1c5505 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:42.284070Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27051" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0dc16-predictor-6b9b847f98-nskqq to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:42Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c68513c6bc3e29 namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: fca312fe-c266-4b19-be4d-c1fcde21ff11 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0dc16-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:42Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c68513d1a6d70a namespace: kserve-ci-e2e-test resourceVersion: "27071" uid: 50a95441-02ae-43a6-a6b4-93f278198b1b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27062" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:43Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c68514046edf09 namespace: kserve-ci-e2e-test resourceVersion: "27088" uid: e82022ef-8ec8-4087-b76a-b8674bd661ee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c6851405db0d8b namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: baff63c4-e042-4219-96ae-f28def5b0883 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c6851408ac39da namespace: kserve-ci-e2e-test resourceVersion: "27092" uid: 5f671a0d-4a67-44b0-bcc3-2584bd704bfc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c6851409195966 namespace: kserve-ci-e2e-test resourceVersion: "27093" uid: 65c711c3-408b-46e0-b84e-01dbd2088317 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c685140922fe84 namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: a355c84b-25bd-4e96-8367-4df3b3ea24f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c685140c039d97 namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: 0f0cf645-8258-4a48-924c-589ee80ec633 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c685140c73c143 namespace: kserve-ci-e2e-test resourceVersion: "27096" uid: 05ef6885-609f-44e5-be66-b8627114d543 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T17:44:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:21Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c685146a08377b namespace: kserve-ci-e2e-test resourceVersion: "27612" uid: a3ab65d0-b559-4674-9448-eb943044bc24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T18:01:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:01:01Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c68605cf3a495b namespace: kserve-ci-e2e-test resourceVersion: "34737" uid: 234978c8-ded9-4e98-9966-db2c8c5db170 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 2324128f-28ce-4991-aaa2-03ce64f72e72 kind: Event lastTimestamp: "2026-07-28T18:01:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:01:01Z" name: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq.18c68605cf3affaa namespace: kserve-ci-e2e-test resourceVersion: "34738" uid: 9137fe9f-2a75-4903-ab5a-fde4b09cdef8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0dc16-predictor-6b9b847f98 namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: 601875ad-c9ec-43b7-a403-5dac601044dd kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: 'Created pod: error-404-isvc-0dc16-predictor-6b9b847f98-nskqq' metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:42Z" name: error-404-isvc-0dc16-predictor-6b9b847f98.18c68513c628b1b7 namespace: kserve-ci-e2e-test resourceVersion: "27052" uid: 77ebb65a-5cfe-40e4-ba09-154155ed8501 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 3d939467-7f8d-4d81-a1a8-66b40d1df436 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: Scaled up replica set error-404-isvc-0dc16-predictor-6b9b847f98 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:42Z" name: error-404-isvc-0dc16-predictor.18c68513c43dad14 namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: 97a1ce52-0e9a-40d1-8886-68a440b3baa7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27043" uid: dbfb8ddd-f51d-4ba6-a5cb-761b95a17533 kind: Event lastTimestamp: "2026-07-28T17:44:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:12Z" name: error-404-isvc-0dc16-predictor.18c6851742b312c0 namespace: kserve-ci-e2e-test resourceVersion: "27550" uid: 3721f859-d157-4f28-9fda-c7988478b1b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27043" uid: dbfb8ddd-f51d-4ba6-a5cb-761b95a17533 kind: Event lastTimestamp: "2026-07-28T17:44:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:12Z" name: error-404-isvc-0dc16-predictor.18c6851743210ad8 namespace: kserve-ci-e2e-test resourceVersion: "27551" uid: c8e4c2cb-3e1c-421c-9450-9846e389a072 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:44:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27385" uid: dbfb8ddd-f51d-4ba6-a5cb-761b95a17533 kind: Event lastTimestamp: "2026-07-28T17:45:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:12Z" name: error-404-isvc-0dc16-predictor.18c6851e4139812a namespace: kserve-ci-e2e-test resourceVersion: "28130" uid: 5f262af6-1b28-4b22-9e36-ada7e859ee3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:44:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27385" uid: dbfb8ddd-f51d-4ba6-a5cb-761b95a17533 kind: Event lastTimestamp: "2026-07-28T17:45:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:12Z" name: error-404-isvc-0dc16-predictor.18c6851e41b3a8ad namespace: kserve-ci-e2e-test resourceVersion: "28131" uid: 84b72b36-11e3-479a-b66d-d0c2b72b436e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: 5e6d66d2-b530-4725-ab21-e0c3ded94251 kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:31Z" name: error-404-isvc-0dc16.18c68513c4438801 namespace: kserve-ci-e2e-test resourceVersion: "27711" uid: 0f3c9c88-cc3b-4d5a-9264-b7e9e322e6e4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: 5e6d66d2-b530-4725-ab21-e0c3ded94251 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: 'Failed to update status for InferenceService "error-404-isvc-0dc16": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0dc16": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:42Z" name: error-404-isvc-0dc16.18c68513c4c3eb6c namespace: kserve-ci-e2e-test resourceVersion: "27048" uid: fe6646d7-b069-4311-881a-9f43d8cc1e37 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: 5e6d66d2-b530-4725-ab21-e0c3ded94251 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0dc16": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:42Z" name: error-404-isvc-0dc16.18c68513c4c48de0 namespace: kserve-ci-e2e-test resourceVersion: "27049" uid: 3cf33bd4-f800-40be-8ad0-3e9a2d7ca6d7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "27703" uid: 5e6d66d2-b530-4725-ab21-e0c3ded94251 kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: InferenceService [error-404-isvc-0dc16] is Ready metadata: creationTimestamp: "2026-07-28T17:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:31Z" name: error-404-isvc-0dc16.18c6851f261234b1 namespace: kserve-ci-e2e-test resourceVersion: "27705" uid: d3b72cbc-e22f-48c9-9ea4-4c68c64887ef reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:42:46.325015Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:42:46Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c68506bf50df96 namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: 11d06532-7b61-41e3-aa79-be95e8384451 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-11994-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c68506cb8ffa33 namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: a6b11133-e2cd-483a-ac01-eb4a10d05019 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26470" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:42:47Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:42:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:42:47Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c68506fc70914e namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 69d7cd4f-8b36-40ee-80aa-b498252cc84d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:42:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:42:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:47Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c68506fdbb2292 namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: 232b8026-7dae-45aa-b91f-814504746a51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:42:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:42:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:47Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c6850700833278 namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: d9802c8f-7a4d-435b-a760-3ec8ae69b7a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:42:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:42:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:47Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c6850700eed699 namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: c457537e-55e1-47d7-9f55-992c4a8abe9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:42:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:42:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:47Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c6850700f7eac7 namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: 011563ca-107d-4e2a-98b8-f4fcb7bad347 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:42:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:42:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:47Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c6850703ce7387 namespace: kserve-ci-e2e-test resourceVersion: "26508" uid: 919c82d1-9d5e-4c3a-affc-04f7ff69f780 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:42:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:42:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:47Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c685070450f12e namespace: kserve-ci-e2e-test resourceVersion: "26509" uid: a230adbd-92b2-418f-8c83-6fb90047ef06 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:42:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:43:24Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:24Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c68507511bc2ab namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: 9a4b8363-a399-4957-88db-c381584a83c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:56Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c685171a2a4635 namespace: kserve-ci-e2e-test resourceVersion: "27331" uid: 2f5723e0-4ae3-4599-8339-c5d7e1eb6d3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:56Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c685171a2af1cd namespace: kserve-ci-e2e-test resourceVersion: "27333" uid: a46e9e4e-4b18-471b-98a0-ce89a4be8a35 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 470a255e-f2cb-4500-8b66-5d4e26766e9e kind: Event lastTimestamp: "2026-07-28T17:44:00Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-28T17:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:00Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv.18c68518150c1662 namespace: kserve-ci-e2e-test resourceVersion: "27445" uid: 6679ce0b-eb7a-4020-9621-2318e0499e57 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-11994-predictor-5d8c7b5cfd namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: 02f4da80-3466-41b0-aac8-f1d5175f27f7 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: 'Created pod: error-404-isvc-11994-predictor-5d8c7b5cfd-kxqxv' metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:46Z" name: error-404-isvc-11994-predictor-5d8c7b5cfd.18c68506bedd744b namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: 509ac1fb-eab7-47e2-8b1d-233a5547708b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26446" uid: ecfd5e1d-a2aa-450f-aa9a-81cce57369e6 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Scaled up replica set error-404-isvc-11994-predictor-5d8c7b5cfd from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:46Z" name: error-404-isvc-11994-predictor.18c68506bcee9e20 namespace: kserve-ci-e2e-test resourceVersion: "26452" uid: b7351cdf-3920-471e-b953-92377c7bb73d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26449" uid: f73a982b-4b1f-4c22-a825-63c5cc9b7168 kind: Event lastTimestamp: "2026-07-28T17:43:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:31Z" name: error-404-isvc-11994-predictor.18c6850a3b757e2a namespace: kserve-ci-e2e-test resourceVersion: "26857" uid: a941b151-0077-4b7b-8753-d15dd7f3c9e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26449" uid: f73a982b-4b1f-4c22-a825-63c5cc9b7168 kind: Event lastTimestamp: "2026-07-28T17:43:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:31Z" name: error-404-isvc-11994-predictor.18c6850a3bdae7b3 namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 6b649fd9-f197-44d3-a2c1-04d7b8600557 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: f73a982b-4b1f-4c22-a825-63c5cc9b7168 kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:46Z" name: error-404-isvc-11994-predictor.18c6850dba9d9077 namespace: kserve-ci-e2e-test resourceVersion: "27164" uid: 6ac5060b-2f62-425c-b1e4-ed72f5a91b30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: f73a982b-4b1f-4c22-a825-63c5cc9b7168 kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:46Z" name: error-404-isvc-11994-predictor.18c6850dbaeb8bd6 namespace: kserve-ci-e2e-test resourceVersion: "27165" uid: c1342313-c3de-4b79-999f-cdb77c5799d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11994 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 9d8c8946-d94b-43ca-9826-18dbd9a24bd4 kind: Event lastTimestamp: "2026-07-28T17:43:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:34Z" name: error-404-isvc-11994.18c68506bcf2fbc0 namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: 0afd5668-4bda-4ede-871f-3951f1747dab reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11994 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 9d8c8946-d94b-43ca-9826-18dbd9a24bd4 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-11994": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-11994": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:42:46Z" name: error-404-isvc-11994.18c68506bd773ba3 namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: 5161c8d1-8dc9-49c4-8ed3-ea1dea8678f2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11994 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 9d8c8946-d94b-43ca-9826-18dbd9a24bd4 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-11994": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:42:46Z" name: error-404-isvc-11994.18c68506bd77a1f8 namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: 746816fc-a29d-450f-be45-08959b85699e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11994 namespace: kserve-ci-e2e-test resourceVersion: "26898" uid: 9d8c8946-d94b-43ca-9826-18dbd9a24bd4 kind: Event lastTimestamp: "2026-07-28T17:43:34Z" message: InferenceService [error-404-isvc-11994] is Ready metadata: creationTimestamp: "2026-07-28T17:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:34Z" name: error-404-isvc-11994.18c685120d6e981e namespace: kserve-ci-e2e-test resourceVersion: "26899" uid: 7f4e118e-88af-4e16-901b-2787e9955a88 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:24:02.097802Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17149" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-211cd-predictor-7596487544-mvkc7 to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c68400fe26ec90 namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: 112b5382-fdab-4bdc-a874-4cbc76ba7efe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17184" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c684011858d3c3 namespace: kserve-ci-e2e-test resourceVersion: "17221" uid: b9675ff7-ae90-4cba-9fd4-3e9a79613e88 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c684011a2c5b82 namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 903518e3-9bc0-47ca-9df3-2ab98d6e3082 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c684011d27f1e5 namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: de7a673d-e199-4280-be21-50e2fd629dbb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c684011d9b2cc4 namespace: kserve-ci-e2e-test resourceVersion: "17226" uid: 8d16cb3a-f1f0-4af9-b0a9-3876bab8555f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c684011da3bd14 namespace: kserve-ci-e2e-test resourceVersion: "17227" uid: 53a50162-a5e8-4dc5-b386-91e0595ecf48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c684012082364d namespace: kserve-ci-e2e-test resourceVersion: "17229" uid: ee523b94-5914-4875-890b-14a2cd2df2a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c6840120e6a038 namespace: kserve-ci-e2e-test resourceVersion: "17230" uid: 6d75ae02-08d0-4413-886a-1f5ed0ac3698 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:33:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:19Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c684015986c7f6 namespace: kserve-ci-e2e-test resourceVersion: "21563" uid: b05e233a-5b9b-435d-aed0-8d3b132d957b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:33:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:16Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c68482277c780c namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 6b4861f1-b53a-4ff0-af51-4c8704c3bbfb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:33:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:16Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c68482277ce3da namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 329c934f-a50d-4250-8b83-ecf47ddcc4c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-211cd-predictor-7596487544-mvkc7 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 57c14be8-6f23-4efd-a099-ec50390dc451 kind: Event lastTimestamp: "2026-07-28T17:33:19Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:19Z" name: error-404-isvc-211cd-predictor-7596487544-mvkc7.18c68482cdfd18fe namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: 77716b3d-75e5-43ee-951f-e14ea8a0df24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-211cd-predictor-7596487544 namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: 3113715a-6265-407f-b717-8ed43426301a kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: 'Created pod: error-404-isvc-211cd-predictor-7596487544-mvkc7' metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor-7596487544.18c68400fd99a615 namespace: kserve-ci-e2e-test resourceVersion: "17154" uid: 5317a324-a8e8-41b2-9368-a8016c90c5c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17138" uid: 7140e387-89c1-414a-8471-a5af97deb901 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: Scaled up replica set error-404-isvc-211cd-predictor-7596487544 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:02Z" name: error-404-isvc-211cd-predictor.18c68400fba6ef8f namespace: kserve-ci-e2e-test resourceVersion: "17142" uid: b5caac8a-2b9c-4d06-af13-8522d7fcee62 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:24:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17139" uid: 30ef5c02-2435-49e5-875a-d20fc8a438cd kind: Event lastTimestamp: "2026-07-28T17:24:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:24:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:47Z" name: error-404-isvc-211cd-predictor.18c684047a8b0dc4 namespace: kserve-ci-e2e-test resourceVersion: "17626" uid: 6e9a7b5c-5116-4e20-bc46-2f113fe9a993 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:24:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17139" uid: 30ef5c02-2435-49e5-875a-d20fc8a438cd kind: Event lastTimestamp: "2026-07-28T17:24:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:24:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:47Z" name: error-404-isvc-211cd-predictor.18c684047b6c014f namespace: kserve-ci-e2e-test resourceVersion: "17628" uid: fd189eeb-1b1d-4747-8799-d0ef0b534062 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:24:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17389" uid: 30ef5c02-2435-49e5-875a-d20fc8a438cd kind: Event lastTimestamp: "2026-07-28T17:25:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:24:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:25:17Z" name: error-404-isvc-211cd-predictor.18c68407fa443cbc namespace: kserve-ci-e2e-test resourceVersion: "17901" uid: 162daa11-7324-448e-b2a9-9661dbfa2b9e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:24:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17389" uid: 30ef5c02-2435-49e5-875a-d20fc8a438cd kind: Event lastTimestamp: "2026-07-28T17:25:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:24:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:25:17Z" name: error-404-isvc-211cd-predictor.18c68407faaf80d2 namespace: kserve-ci-e2e-test resourceVersion: "17902" uid: 6efab9c2-9196-4b18-bb26-f399cff247fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-211cd namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: ee4a00c9-2c5b-4f0c-ab29-a8bd187d956c kind: Event lastTimestamp: "2026-07-28T17:24:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:24:49Z" name: error-404-isvc-211cd.18c68400fbda110c namespace: kserve-ci-e2e-test resourceVersion: "17654" uid: 17b2a9a0-510e-4506-af6e-7846dd002434 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-211cd namespace: kserve-ci-e2e-test resourceVersion: "17651" uid: ee4a00c9-2c5b-4f0c-ab29-a8bd187d956c kind: Event lastTimestamp: "2026-07-28T17:24:49Z" message: InferenceService [error-404-isvc-211cd] is Ready metadata: creationTimestamp: "2026-07-28T17:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:24:49Z" name: error-404-isvc-211cd.18c6840c15635165 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: a4e0778e-1244-4ebf-8962-b020b8114d1a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:23:02.184606Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-391b7-predictor-694d665486-h66x4 to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c683f30b0c139a namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 9a82d0ac-1b44-4ca1-a22c-9722a2134616 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c683f32497533a namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: 07ae2697-98d2-4527-b21e-01f0c13a81c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c683f325f8650b namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: b3104589-ee3c-4be5-ab72-d4d6c884ed25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c683f32958fc80 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 10f37cc5-5e8b-4702-9cb6-9fb23d53ccc0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c683f329e2d7a7 namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: 9eb3c857-ae6c-4cfb-8479-4d2626568131 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c683f329ed1e12 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: f73ae506-82d2-4c35-8076-8839ef199b4e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c683f32d61ab8d namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 0071248b-4fab-4974-85ea-acc811d13e9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c683f32dc9619e namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 0d82ed5d-914f-469a-b8dd-5cced644e246 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:32:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:20Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c683f38bdca2ff namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 907e75b3-36cc-44ae-b57b-6dd1f7ee0d21 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c684743fb95139 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 7254aab2-5729-48b7-95d6-407aa94f9fe0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c684743fb9fa6f namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 591e14e4-0ba4-4e9d-96b6-d680acc4ba5a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-391b7-predictor-694d665486-h66x4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 07e2a171-286b-44fb-97ce-d1174e076de7 kind: Event lastTimestamp: "2026-07-28T17:32:20Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:20Z" name: error-404-isvc-391b7-predictor-694d665486-h66x4.18c684750075870c namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: ca84dbcf-d025-4bcd-bdb6-0d20be2deb28 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-391b7-predictor-694d665486 namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 5ec2f97b-ac5a-4bbc-8b49-7a31db3e577e kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: 'Created pod: error-404-isvc-391b7-predictor-694d665486-h66x4' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor-694d665486.18c683f30a7c92b1 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: ca291dd8-5971-456d-90cd-ff2d5c073613 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: d55f27bd-5817-4a5b-bb65-45ba39dc37af kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Scaled up replica set error-404-isvc-391b7-predictor-694d665486 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7-predictor.18c683f308df535e namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 70bcae6b-9f5c-49a6-a9d5-b1ff88bcb233 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:23:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: 1e27d503-bee0-4412-9b1a-89a75d619815 kind: Event lastTimestamp: "2026-07-28T17:23:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:47Z" name: error-404-isvc-391b7-predictor.18c683f687cb3d1e namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: a06c91c3-9283-4537-b95d-778e358a771c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:23:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: 1e27d503-bee0-4412-9b1a-89a75d619815 kind: Event lastTimestamp: "2026-07-28T17:23:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:47Z" name: error-404-isvc-391b7-predictor.18c683f688407546 namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 86197538-a7bf-4979-b2db-e56fbd117cce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 1e27d503-bee0-4412-9b1a-89a75d619815 kind: Event lastTimestamp: "2026-07-28T17:24:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:17Z" name: error-404-isvc-391b7-predictor.18c6840104766f15 namespace: kserve-ci-e2e-test resourceVersion: "17398" uid: 984200ec-0848-43a1-b898-72093d2d278a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 1e27d503-bee0-4412-9b1a-89a75d619815 kind: Event lastTimestamp: "2026-07-28T17:24:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:17Z" name: error-404-isvc-391b7-predictor.18c68401051271e3 namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: 6998ee7b-9f70-4609-b074-0d80f10bab99 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-391b7 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 75a07449-ddca-414e-93a0-56489b8b4c66 kind: Event lastTimestamp: "2026-07-28T17:23:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:50Z" name: error-404-isvc-391b7.18c683f308e589b5 namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: d6e53f33-3a26-47e8-9e8f-fda15db892ce reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-391b7 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 75a07449-ddca-414e-93a0-56489b8b4c66 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-391b7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-391b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7.18c683f30968ee59 namespace: kserve-ci-e2e-test resourceVersion: "16442" uid: 228d3500-bf43-4657-927b-7fb658b5e801 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-391b7 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 75a07449-ddca-414e-93a0-56489b8b4c66 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-391b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-391b7.18c683f30969288c namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: 1244d0ed-74ee-46a0-8958-64ebfd235555 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-391b7 namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: 75a07449-ddca-414e-93a0-56489b8b4c66 kind: Event lastTimestamp: "2026-07-28T17:23:50Z" message: InferenceService [error-404-isvc-391b7] is Ready metadata: creationTimestamp: "2026-07-28T17:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:50Z" name: error-404-isvc-391b7.18c683fe48ea11e6 namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 2d842be0-d760-45a0-8936-3cb09da60def reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:17.157609Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c6847442042622 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: b1918b21-6d38-4e9c-8ccf-09ec969f95b6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c684745b7a9a79 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 00f83c23-0fd1-4ee5-97b3-d85d1e14dc58 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c684745d2b7302 namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: 1415f348-dff8-4377-b52e-e69674e8c8d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c6847460500441 namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: 7eefd69c-06b3-4982-aa05-a2e61a9a11ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c6847460bbfaaa namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: c6f8f49a-2991-416d-8877-e9bc279f6614 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c6847460c69972 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 45524ba0-74b6-4664-b3af-07f3e7e961e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c68474637cc4e9 namespace: kserve-ci-e2e-test resourceVersion: "20891" uid: c37248e9-b0a6-4a35-b88a-a6b737dc5238 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c6847463de57a3 namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: 3d224344-e2ac-42bb-b865-391952a0b120 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:33:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:33Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c684746bfc799f namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9f6bd0af-30f1-449f-883d-9163cc3d1e4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c6848585133c26 namespace: kserve-ci-e2e-test resourceVersion: "21702" uid: 3b1adef4-1d5c-4db8-9610-04244b8579ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c684858513cb46 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 9e070948-f406-4ab0-81f1-f31a9f34fc1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 6be7aedd-9954-40f7-a018-8d36a99b4f71 kind: Event lastTimestamp: "2026-07-28T17:33:33Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:33Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj.18c684861e2b47e1 namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: e855bcbd-3344-4a85-890c-2235fc3bc0cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4a23a-predictor-6b5bbf5776 namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: fa3bf738-6e0e-4554-ad6d-010593f42762 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: 'Created pod: error-404-isvc-4a23a-predictor-6b5bbf5776-rnjsj' metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor-6b5bbf5776.18c68474418c7334 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 503be149-f25e-4b35-8dd5-c107835fd246 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: fe384610-d084-4cd6-bb57-c55846faea33 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Scaled up replica set error-404-isvc-4a23a-predictor-6b5bbf5776 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a-predictor.18c684743e6aff4a namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: a4661424-2095-4922-842a-5df17b315e20 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: fba40b90-1e3a-4f80-89b5-fda8a4e69b22 kind: Event lastTimestamp: "2026-07-28T17:33:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:02Z" name: error-404-isvc-4a23a-predictor.18c68477bcf89303 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 0d827909-7b29-4783-acba-28d0779e17f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: fba40b90-1e3a-4f80-89b5-fda8a4e69b22 kind: Event lastTimestamp: "2026-07-28T17:33:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:02Z" name: error-404-isvc-4a23a-predictor.18c68477bd5f3e2e namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 8049d75d-7b36-45bd-8efd-3717acb30b7a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: fba40b90-1e3a-4f80-89b5-fda8a4e69b22 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-4a23a-predictor.18c6847b3c80de4f namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: 6c5258f9-4d11-4c59-a390-a651371d197b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:32:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: fba40b90-1e3a-4f80-89b5-fda8a4e69b22 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-4a23a-predictor.18c6847b3d010c6f namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: d14fa9cd-8282-45db-92af-ed979dccded0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a23a namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: f9ebf5c7-b6a2-4fe2-83e6-c28dec52c0ad kind: Event lastTimestamp: "2026-07-28T17:33:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:03Z" name: error-404-isvc-4a23a.18c684743eca1b7a namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: d1c38024-92b4-45de-aab1-e8d6efc404ec reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a23a namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: f9ebf5c7-b6a2-4fe2-83e6-c28dec52c0ad kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-4a23a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4a23a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a.18c684743fedab75 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 07a9cbc9-fcc2-403e-b652-1f54a22e6514 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a23a namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: f9ebf5c7-b6a2-4fe2-83e6-c28dec52c0ad kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4a23a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:17Z" name: error-404-isvc-4a23a.18c684743fedfafb namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: e8c2550b-4141-473d-b3c7-98dd5a1277e3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a23a namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: f9ebf5c7-b6a2-4fe2-83e6-c28dec52c0ad kind: Event lastTimestamp: "2026-07-28T17:33:03Z" message: InferenceService [error-404-isvc-4a23a] is Ready metadata: creationTimestamp: "2026-07-28T17:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:03Z" name: error-404-isvc-4a23a.18c6847f2766c200 namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: 59a2b9fd-e01b-4fc5-b928-d02418fd012b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:56.677684Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27354" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-824fb-predictor-8dbccd486-x4jf6 to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:56Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c6851720a967ba namespace: kserve-ci-e2e-test resourceVersion: "27358" uid: bafd6628-9ea8-4ad7-abe1-4eb069521830 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-824fb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:56Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c685172c5cd2ed namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 748df857-2bc3-45a3-badd-fd26d5a57909 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27365" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:57Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c685175e9e1c97 namespace: kserve-ci-e2e-test resourceVersion: "27396" uid: c48b1b51-d0c2-41c9-bf1d-0250274cf022 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c6851760bae665 namespace: kserve-ci-e2e-test resourceVersion: "27399" uid: 7af55ddc-6e10-47bf-b77d-bd76983da574 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c6851763ec82a0 namespace: kserve-ci-e2e-test resourceVersion: "27400" uid: 7c7e4819-fc6e-4974-932c-02656ac1ac8f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c68517645bfb11 namespace: kserve-ci-e2e-test resourceVersion: "27402" uid: 57aa16d5-beef-490a-a298-89f4c3c8c784 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c6851764647ac1 namespace: kserve-ci-e2e-test resourceVersion: "27403" uid: ce6ab586-a191-4986-922e-9361feaa16f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c6851767779737 namespace: kserve-ci-e2e-test resourceVersion: "27406" uid: f77eb632-decf-4b67-b14f-919227cc39fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c6851767e75fbb namespace: kserve-ci-e2e-test resourceVersion: "27407" uid: 3cfc1828-58ce-4f63-83f5-c735007681f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:44:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:35Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c68517b07a0c84 namespace: kserve-ci-e2e-test resourceVersion: "27745" uid: 19f188e7-4eba-4d44-b230-3dee14dab248 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:53:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:11Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c6859846fe217f namespace: kserve-ci-e2e-test resourceVersion: "31481" uid: d248e838-589a-46f5-b4e2-1e891406832d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 2e45aaf6-2b1c-48fb-8e95-178b3f25a924 kind: Event lastTimestamp: "2026-07-28T17:53:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:11Z" name: error-404-isvc-824fb-predictor-8dbccd486-x4jf6.18c6859846fea106 namespace: kserve-ci-e2e-test resourceVersion: "31482" uid: 5c560af3-c5ba-44ea-92a3-624b07523590 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-824fb-predictor-8dbccd486 namespace: kserve-ci-e2e-test resourceVersion: "27347" uid: 1714feec-957b-45cb-a535-345ff9cee36c kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: 'Created pod: error-404-isvc-824fb-predictor-8dbccd486-x4jf6' metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:56Z" name: error-404-isvc-824fb-predictor-8dbccd486.18c685172061bd2e namespace: kserve-ci-e2e-test resourceVersion: "27356" uid: 4a250f61-c198-48d5-b478-aa36ef5b0944 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: f28e968a-ae1c-4aa8-b8d5-f31d6ce89710 kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: Scaled up replica set error-404-isvc-824fb-predictor-8dbccd486 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:56Z" name: error-404-isvc-824fb-predictor.18c685171e9afde8 namespace: kserve-ci-e2e-test resourceVersion: "27349" uid: 77ff912d-c818-4a48-ac59-942030bc10a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27344" uid: 92db615c-bf00-446e-aa7b-0905b69b908e kind: Event lastTimestamp: "2026-07-28T17:44:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:26Z" name: error-404-isvc-824fb-predictor.18c6851a9d34c8f1 namespace: kserve-ci-e2e-test resourceVersion: "27655" uid: fb1217cb-91e3-43fa-b5b0-b8dece279ae4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27344" uid: 92db615c-bf00-446e-aa7b-0905b69b908e kind: Event lastTimestamp: "2026-07-28T17:44:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:26Z" name: error-404-isvc-824fb-predictor.18c6851a9da21c4e namespace: kserve-ci-e2e-test resourceVersion: "27656" uid: 442a8d8c-bdc1-47cf-8b26-b253aea9764e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27538" uid: 92db615c-bf00-446e-aa7b-0905b69b908e kind: Event lastTimestamp: "2026-07-28T17:45:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:11Z" name: error-404-isvc-824fb-predictor.18c685219b254a44 namespace: kserve-ci-e2e-test resourceVersion: "28120" uid: 6e8820f7-2f1a-4d1d-adea-fb3656403f6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27538" uid: 92db615c-bf00-446e-aa7b-0905b69b908e kind: Event lastTimestamp: "2026-07-28T17:45:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:11Z" name: error-404-isvc-824fb-predictor.18c685219b8f80ee namespace: kserve-ci-e2e-test resourceVersion: "28121" uid: 90b5b42c-301e-4c37-8703-5a7520c3bf53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-824fb namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 9fa6f1a2-08dd-44d6-8336-ff4745ccb1f2 kind: Event lastTimestamp: "2026-07-28T17:44:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:45Z" name: error-404-isvc-824fb.18c685171e3c33ee namespace: kserve-ci-e2e-test resourceVersion: "27880" uid: 3d9e5f7a-9158-47b3-b822-aab41c614c8f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-824fb namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 9fa6f1a2-08dd-44d6-8336-ff4745ccb1f2 kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-824fb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-824fb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:56Z" name: error-404-isvc-824fb.18c685171eab578f namespace: kserve-ci-e2e-test resourceVersion: "27350" uid: f861be2e-7029-444b-8ad3-0f2ebb29f0ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-824fb namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 9fa6f1a2-08dd-44d6-8336-ff4745ccb1f2 kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-824fb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:56Z" name: error-404-isvc-824fb.18c685171eab9d72 namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: 78a571a4-9353-4a90-898a-56580eb14be3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-824fb namespace: kserve-ci-e2e-test resourceVersion: "27878" uid: 9fa6f1a2-08dd-44d6-8336-ff4745ccb1f2 kind: Event lastTimestamp: "2026-07-28T17:44:45Z" message: InferenceService [error-404-isvc-824fb] is Ready metadata: creationTimestamp: "2026-07-28T17:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:45Z" name: error-404-isvc-824fb.18c685226c7c99d3 namespace: kserve-ci-e2e-test resourceVersion: "27879" uid: af625b77-3eb0-47d3-b69d-fbbc9ba19c39 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:34:27.370980Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:34:27Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c6849293566cb7 namespace: kserve-ci-e2e-test resourceVersion: "22447" uid: dc05ee9f-d8cc-4f2a-a5b8-ee764779a138 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a513a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c684929d59e7d2 namespace: kserve-ci-e2e-test resourceVersion: "22462" uid: a17db652-544b-4990-bdf7-b4c8946299be reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22454" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:34:28Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:34:28Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c68492d0d25b00 namespace: kserve-ci-e2e-test resourceVersion: "22483" uid: 78e07db0-b4d7-4389-8fe0-e0e6bfbd5fea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:34:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:28Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c68492d24137d8 namespace: kserve-ci-e2e-test resourceVersion: "22485" uid: face2ec9-3fc3-4b28-8254-733a7accb010 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:34:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:28Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c68492d537db99 namespace: kserve-ci-e2e-test resourceVersion: "22486" uid: f8c1ac04-9c1d-462d-939a-597da7552170 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:34:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:28Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c68492d5ac53f5 namespace: kserve-ci-e2e-test resourceVersion: "22487" uid: 8a446bd0-894b-4814-b8f4-70d91315a736 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:34:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:28Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c68492d5b6b241 namespace: kserve-ci-e2e-test resourceVersion: "22488" uid: 01339ba1-1cc4-4c34-a509-86c19332db82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:34:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:28Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c68492d924ed0d namespace: kserve-ci-e2e-test resourceVersion: "22490" uid: 3dd11923-9f26-402e-9402-0119606d2195 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:34:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:28Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c68492d99bab71 namespace: kserve-ci-e2e-test resourceVersion: "22491" uid: d4bf8c70-1213-4cc5-bb4a-4b8444ceaa62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:35:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:05Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c6849311e178ca namespace: kserve-ci-e2e-test resourceVersion: "22846" uid: fcbb3b27-7e8d-4c19-8553-09a412e2c7aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:42Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c68513b877f096 namespace: kserve-ci-e2e-test resourceVersion: "26990" uid: f0c3fc7d-d5e0-4f9d-acd5-b5c555cdf525 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: a3f004e1-2f43-487a-9288-3d83749b7295 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:42Z" name: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8.18c68513b8787980 namespace: kserve-ci-e2e-test resourceVersion: "26992" uid: 1a0efe91-9bfe-4aa9-aabf-ebef3ed269bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a513a-predictor-d745bdcc6 namespace: kserve-ci-e2e-test resourceVersion: "22436" uid: 22b8106b-d08f-459e-b76e-d89750db6931 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: 'Created pod: error-404-isvc-a513a-predictor-d745bdcc6-9xwj8' metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:27Z" name: error-404-isvc-a513a-predictor-d745bdcc6.18c6849292dc6b8f namespace: kserve-ci-e2e-test resourceVersion: "22444" uid: 95d1d1be-3a52-4518-b061-b10472286f89 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22433" uid: 79e540e6-0f1c-483c-a0d2-f01664df11a2 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Scaled up replica set error-404-isvc-a513a-predictor-d745bdcc6 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:27Z" name: error-404-isvc-a513a-predictor.18c6849291247945 namespace: kserve-ci-e2e-test resourceVersion: "22438" uid: b41a15e6-63f0-430a-9322-5acb5f7a529f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: c8312ae0-b48f-438a-bbd1-f108f670222f kind: Event lastTimestamp: "2026-07-28T17:34:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:42Z" name: error-404-isvc-a513a-predictor.18c684960f6b5954 namespace: kserve-ci-e2e-test resourceVersion: "22660" uid: 1983d3e9-aa73-4743-9ee8-561c3b530867 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: c8312ae0-b48f-438a-bbd1-f108f670222f kind: Event lastTimestamp: "2026-07-28T17:34:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:42Z" name: error-404-isvc-a513a-predictor.18c684960fd2e5cf namespace: kserve-ci-e2e-test resourceVersion: "22662" uid: b35084aa-09c9-483e-92ca-b8e5727ed917 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:34:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22661" uid: c8312ae0-b48f-438a-bbd1-f108f670222f kind: Event lastTimestamp: "2026-07-28T17:35:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:42Z" name: error-404-isvc-a513a-predictor.18c684998e9c0457 namespace: kserve-ci-e2e-test resourceVersion: "23172" uid: c9dc0ea5-0561-4617-aba9-40ac46368219 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:34:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22661" uid: c8312ae0-b48f-438a-bbd1-f108f670222f kind: Event lastTimestamp: "2026-07-28T17:35:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:42Z" name: error-404-isvc-a513a-predictor.18c684998f07ba6b namespace: kserve-ci-e2e-test resourceVersion: "23173" uid: 6f1cc287-13ce-4a9e-aba3-341929f0549c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a513a namespace: kserve-ci-e2e-test resourceVersion: "22423" uid: 71df7c23-141a-4c4e-a691-dc94c32b7bb0 kind: Event lastTimestamp: "2026-07-28T17:35:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:15Z" name: error-404-isvc-a513a.18c6849290d7b68f namespace: kserve-ci-e2e-test resourceVersion: "22934" uid: 2df2a8c0-cb64-4851-927b-d1869c5d2ef7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a513a namespace: kserve-ci-e2e-test resourceVersion: "22423" uid: 71df7c23-141a-4c4e-a691-dc94c32b7bb0 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-a513a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a513a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:27Z" name: error-404-isvc-a513a.18c68492913b7b62 namespace: kserve-ci-e2e-test resourceVersion: "22439" uid: 76a96181-eb19-4f70-ab8c-dcc68ff2b5f7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a513a namespace: kserve-ci-e2e-test resourceVersion: "22423" uid: 71df7c23-141a-4c4e-a691-dc94c32b7bb0 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a513a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:27Z" name: error-404-isvc-a513a.18c68492913bd17f namespace: kserve-ci-e2e-test resourceVersion: "22441" uid: cdffc92e-2cb2-42a6-a85c-b4a3b708d54f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a513a namespace: kserve-ci-e2e-test resourceVersion: "22932" uid: 71df7c23-141a-4c4e-a691-dc94c32b7bb0 kind: Event lastTimestamp: "2026-07-28T17:35:15Z" message: InferenceService [error-404-isvc-a513a] is Ready metadata: creationTimestamp: "2026-07-28T17:35:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:15Z" name: error-404-isvc-a513a.18c6849dcd745620 namespace: kserve-ci-e2e-test resourceVersion: "22933" uid: ad8a0fa4-bba3-46de-854f-2e1b3bc4686d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:21:32.174163Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:21:32Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683de1601c8eb namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 1efe9d34-6e3a-4463-8213-e13fffc75e03 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:21:32Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683de30111f1c namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: a741be7e-debb-4519-aba4-bedeae5505e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683de32517e9d namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: ad33991c-4b78-47da-b71a-1f66dfe384f0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:21:44Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" in 11.38s (11.38s including waiting). Image size: 1333940313 bytes.' metadata: creationTimestamp: "2026-07-28T17:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:44Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683e0d8a65b6a namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: 30cea55e-6153-4672-ba54-a452d2783a24 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:21:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:44Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683e0dc8a31f1 namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: a7722d74-13d5-4623-83f6-1b423311332b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:21:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:44Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683e0dd04f960 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: 88cdbd36-7a71-4f86-a3b9-943a6d3acb40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:21:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:44Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683e0dd12b9e4 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: d1671a18-1ede-489c-ba3e-11a2292d25c9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:21:46Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.225s (2.225s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:46Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683e161bf490e namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: c6826d1b-2a10-4ae2-9ed0-7c11c8027e36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:21:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:46Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683e164f3a885 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 94b9080a-3f1a-46da-84b7-761656c13c01 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:21:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:46Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683e1656a6b24 namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 8d0cd9bd-b4c9-4838-844d-d59a38936ed7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683e188891dce namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 41b0f7c4-f654-4635-b712-bca412b12c5c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683f3045f5dfd namespace: kserve-ci-e2e-test resourceVersion: "16421" uid: ed504e90-6c4c-4bcf-b454-4f8111949cb8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683f3046015cb namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: a3ce662e-7e6f-4953-a2ea-4ac14e5af8c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 8d25ef18-6f62-4906-91bd-63a4828c35d8 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27.18c683f33ac58c5c namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 45abb8e6-8d71-4314-9fff-6895a26c17ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b94d0-predictor-57d9b4447b namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 93ee8e24-fdd2-4131-963c-04233d30ffbb kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: 'Created pod: error-404-isvc-b94d0-predictor-57d9b4447b-cgg27' metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:32Z" name: error-404-isvc-b94d0-predictor-57d9b4447b.18c683de1530e8d6 namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: d6674425-5966-4760-b95c-3ecf25dae911 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: 369160a4-da10-4b05-9f6d-4ed3deca09a9 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Scaled up replica set error-404-isvc-b94d0-predictor-57d9b4447b from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:32Z" name: error-404-isvc-b94d0-predictor.18c683de132f2f3e namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: f3dbf2ce-d4a0-4936-8d44-3007ac2fe5c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:21:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: d3347d18-52c3-44bb-bed8-d6bb520e2811 kind: Event lastTimestamp: "2026-07-28T17:22:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:02Z" name: error-404-isvc-b94d0-predictor.18c683e191ae1934 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: bb74e1e2-bd10-40f6-983e-fc15caaa8e5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:21:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: d3347d18-52c3-44bb-bed8-d6bb520e2811 kind: Event lastTimestamp: "2026-07-28T17:22:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:02Z" name: error-404-isvc-b94d0-predictor.18c683e19254871c namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 1412a5d9-cc6a-4c48-8842-052a256a2dcc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:22:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15655" uid: d3347d18-52c3-44bb-bed8-d6bb520e2811 kind: Event lastTimestamp: "2026-07-28T17:22:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:47Z" name: error-404-isvc-b94d0-predictor.18c683e8907ef083 namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 6fe8b18f-25f4-4035-90e6-61b18f6896f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:22:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15655" uid: d3347d18-52c3-44bb-bed8-d6bb520e2811 kind: Event lastTimestamp: "2026-07-28T17:22:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:47Z" name: error-404-isvc-b94d0-predictor.18c683e89109aad6 namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: 630597ca-931a-4a52-ba4c-d84d4b0d1512 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b94d0 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 098bc2b9-ea5e-41ca-a379-f3a1b96354ab kind: Event lastTimestamp: "2026-07-28T17:22:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:33Z" name: error-404-isvc-b94d0.18c683de132cd3fe namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: c278a8d8-996b-4a3d-90f4-6de8d7fe9f83 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b94d0 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 098bc2b9-ea5e-41ca-a379-f3a1b96354ab kind: Event lastTimestamp: "2026-07-28T17:22:33Z" message: InferenceService [error-404-isvc-b94d0] is Ready metadata: creationTimestamp: "2026-07-28T17:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:33Z" name: error-404-isvc-b94d0.18c683ec44e0d0f4 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: f0cd6c0a-2fe9-44ff-848f-81924fa7bfaa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:31.493332Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21762" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c6848590c53744 namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 9eb33428-96d7-41ba-911d-8dc2862e6b50 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21771" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68485aa962861 namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 97022bbe-6bc7-4b8d-8f19-ca9ad0b3fbee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68485abfaf777 namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: 07488e56-e9f3-4a1d-9ef0-0f42b70732af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68485aedfd06c namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 5e5078b4-7a32-404f-bebf-e6183b4418f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68485af53f79c namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: 08a34cee-d689-4e4f-a3f3-52914cd0b582 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68485af5df120 namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: 50524fa2-e20e-4dbf-a99d-b383cd7e950e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68485b2d8651e namespace: kserve-ci-e2e-test resourceVersion: "21803" uid: 262b0a14-43c1-4870-a675-5ab5a5228416 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68485b35047dd namespace: kserve-ci-e2e-test resourceVersion: "21804" uid: b3c1b3e5-ce8c-4f92-8d28-d0b1edd91cc2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:42:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:49Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68485f77ef104 namespace: kserve-ci-e2e-test resourceVersion: "26529" uid: c324b8b1-975f-48d8-9ba1-2844c01b9360 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68506d90c97e0 namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 8ad9103d-493d-4e82-8351-825a9ac8ebb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c68506d90d0bbd namespace: kserve-ci-e2e-test resourceVersion: "26487" uid: efbb694a-8d1a-44ea-9a15-6700df788892 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 50e675ef-4e44-42ea-87c8-fd22be665c28 kind: Event lastTimestamp: "2026-07-28T17:42:49Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:49Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm.18c685076bdd547e namespace: kserve-ci-e2e-test resourceVersion: "26528" uid: 25dcb5ee-3b15-4da2-9927-1ee85d2c318c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d1fc0-predictor-8b6dd7598 namespace: kserve-ci-e2e-test resourceVersion: "21755" uid: 3163206e-ac21-460a-929b-270e97f6548d kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'Created pod: error-404-isvc-d1fc0-predictor-8b6dd7598-fsjtm' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-d1fc0-predictor-8b6dd7598.18c6848590584db8 namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: 15ff445a-490a-459c-be83-7a8374180cfb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21753" uid: ac5ea7c1-e945-4e46-b73a-494cd5e2432a kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Scaled up replica set error-404-isvc-d1fc0-predictor-8b6dd7598 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-d1fc0-predictor.18c684858ebc2886 namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: 2e89c88d-7593-4178-bcd2-01e99f46d42f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21754" uid: 7f868f72-ab41-4506-8fa4-d5e9be902026 kind: Event lastTimestamp: "2026-07-28T17:34:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:01Z" name: error-404-isvc-d1fc0-predictor.18c684890d53c92b namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 46d78bfb-b6af-4ba2-8054-37395fce92a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21754" uid: 7f868f72-ab41-4506-8fa4-d5e9be902026 kind: Event lastTimestamp: "2026-07-28T17:34:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:01Z" name: error-404-isvc-d1fc0-predictor.18c684890da5a5ac namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: c1191867-9501-4a56-acf8-2c47a9995754 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: 7f868f72-ab41-4506-8fa4-d5e9be902026 kind: Event lastTimestamp: "2026-07-28T17:34:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:46Z" name: error-404-isvc-d1fc0-predictor.18c684900b2862f5 namespace: kserve-ci-e2e-test resourceVersion: "22698" uid: 0c4601b6-4018-47cd-ba59-913ea3d5a3ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: 7f868f72-ab41-4506-8fa4-d5e9be902026 kind: Event lastTimestamp: "2026-07-28T17:34:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:46Z" name: error-404-isvc-d1fc0-predictor.18c684900bba2f5a namespace: kserve-ci-e2e-test resourceVersion: "22699" uid: a500ca12-2983-4c67-afa9-68a4e6e5e8e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: b07a0e0e-8e65-4476-a061-bd8b92dccc96 kind: Event lastTimestamp: "2026-07-28T17:34:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:19Z" name: error-404-isvc-d1fc0.18c684858ec4a297 namespace: kserve-ci-e2e-test resourceVersion: "22284" uid: ce6badf4-9a78-483e-8d86-ca8cf59b9368 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: b07a0e0e-8e65-4476-a061-bd8b92dccc96 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-d1fc0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d1fc0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-d1fc0.18c684858f425cfd namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 60fa83bc-c8f0-40b3-9706-a1ae680d7e4d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: b07a0e0e-8e65-4476-a061-bd8b92dccc96 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d1fc0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:31Z" name: error-404-isvc-d1fc0.18c684858f42b7c3 namespace: kserve-ci-e2e-test resourceVersion: "21761" uid: ea348631-5908-4d7b-9aa4-976266257712 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "22282" uid: b07a0e0e-8e65-4476-a061-bd8b92dccc96 kind: Event lastTimestamp: "2026-07-28T17:34:19Z" message: InferenceService [error-404-isvc-d1fc0] is Ready metadata: creationTimestamp: "2026-07-28T17:34:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:19Z" name: error-404-isvc-d1fc0.18c68490b3147ed1 namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 8a6bded3-9355-4384-934e-c08716fa3501 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:17.052354Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb to ip-10-0-131-184.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c6848234056e81 namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: 7d42ad8a-127f-48f4-8f61-cf46debc7eb2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c684824e350c95 namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: a834af3d-608b-439d-8c82-44aa9fbaf699 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c684824fa0ea03 namespace: kserve-ci-e2e-test resourceVersion: "21519" uid: a567e18a-95d8-411b-904d-dbfd22fde074 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c6848252ad3b3d namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 0b7950fc-c654-442a-8ba2-3951a70171b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c68482531f6c94 namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: 2647f054-17c0-4dad-ab6e-4724583a2af4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c68482532aa7a8 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: cb64b65f-de5f-41a3-832a-ee6e3914b531 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c6848257320e75 namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: 1d536441-1742-4469-a7a7-0a00e4ad27cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c6848257b0a1ea namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: 48035511-f13f-4c32-aa59-56d4ff29b902 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:33:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:55Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c68482b0ab2e75 namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: b50bc3d8-e61a-4ca6-8c24-4479931a756b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c684928864bcf5 namespace: kserve-ci-e2e-test resourceVersion: "22385" uid: 465a1f71-1261-4de5-9c31-673ab0a742ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c684928865cce3 namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: bc62b814-f5ba-4bb9-ac55-6e9bd3598015 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 53cdfbf0-e5fd-4e59-893c-43e83358704b kind: Event lastTimestamp: "2026-07-28T17:34:30Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:30Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb.18c684933907afe5 namespace: kserve-ci-e2e-test resourceVersion: "22512" uid: a053de96-173d-4518-bc09-b6edbc15b68e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-184.ec2.internal source: component: kubelet host: ip-10-0-131-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ef873-predictor-6bdbb99f4d namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: b6bc9abb-6731-443a-bff3-25b630e47c57 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'Created pod: error-404-isvc-ef873-predictor-6bdbb99f4d-plzsb' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor-6bdbb99f4d.18c684823376e80a namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 914295ed-a57a-48d5-b7dd-f5cf539a3422 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 9095c50e-19fe-463f-baf8-58ac7fe8b6b8 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: Scaled up replica set error-404-isvc-ef873-predictor-6bdbb99f4d from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873-predictor.18c68482315eb356 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 291c3c51-e5d4-4ae7-b0e7-3966ae015d44 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: a87fa5e2-5643-4235-a101-f0b28a55692a kind: Event lastTimestamp: "2026-07-28T17:34:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:02Z" name: error-404-isvc-ef873-predictor.18c68485af8e5714 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: a8e6a3b1-3fbb-4968-b870-df980429e55e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: a87fa5e2-5643-4235-a101-f0b28a55692a kind: Event lastTimestamp: "2026-07-28T17:34:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:02Z" name: error-404-isvc-ef873-predictor.18c68485aff2fcfd namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: c2ef33bc-0bd0-4bb5-bd9b-df5ff3353c11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: a87fa5e2-5643-4235-a101-f0b28a55692a kind: Event lastTimestamp: "2026-07-28T17:34:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:17Z" name: error-404-isvc-ef873-predictor.18c684902c9594b0 namespace: kserve-ci-e2e-test resourceVersion: "22247" uid: fc63e2de-d05d-46cf-aaec-2d78dc60cab0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: a87fa5e2-5643-4235-a101-f0b28a55692a kind: Event lastTimestamp: "2026-07-28T17:34:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:17Z" name: error-404-isvc-ef873-predictor.18c684902d23a083 namespace: kserve-ci-e2e-test resourceVersion: "22249" uid: f8e75392-fd19-45fe-89d9-091447acae0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ef873 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: efbe3d0b-527b-4f24-a573-79a50d344e19 kind: Event lastTimestamp: "2026-07-28T17:34:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:05Z" name: error-404-isvc-ef873.18c68482314eee3f namespace: kserve-ci-e2e-test resourceVersion: "22128" uid: a35b3f91-a9f4-4a10-9c8d-9c7e2c037ad7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ef873 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: efbe3d0b-527b-4f24-a573-79a50d344e19 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-ef873": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ef873": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873.18c6848231f3cc23 namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: f9074256-b144-4835-9b87-2adc08ae526e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ef873 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: efbe3d0b-527b-4f24-a573-79a50d344e19 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ef873": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:17Z" name: error-404-isvc-ef873.18c6848231f4170b namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: d8c302cc-b63f-49c7-ae07-9fb1dd3989a7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ef873 namespace: kserve-ci-e2e-test resourceVersion: "22126" uid: efbe3d0b-527b-4f24-a573-79a50d344e19 kind: Event lastTimestamp: "2026-07-28T17:34:05Z" message: InferenceService [error-404-isvc-ef873] is Ready metadata: creationTimestamp: "2026-07-28T17:34:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:05Z" name: error-404-isvc-ef873.18c6848d6cee9013 namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: 9f49f894-abe6-46ef-aa50-cc4be82f322d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:21:31.905103Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-84c58bd689-f96qc to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:21:31Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683de05f83269 namespace: kserve-ci-e2e-test resourceVersion: "15367" uid: 944e6a00-dd86-4a49-a050-0c900477052b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683de1f9e1fa1 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: f27f5273-1994-4305-8130-b74936429641 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" already present on machine metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683de2123834c namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: bf309a08-fa99-481c-93e4-b359586dfb9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683de24ac5171 namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 3007c760-0e6f-4318-aabf-98976d749d02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683de2520dc20 namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 3b665cda-0f34-49df-9bd0-d45d80a68c27 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:21:37Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:21:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:37Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683df47c3db3f namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: fbd87262-1555-4f60-910a-924eda43748e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:22:05Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" in 28.622s (28.622s including waiting). Image size: 1555722147 bytes.' metadata: creationTimestamp: "2026-07-28T17:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:05Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683e5f1cb9bfa namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 358ce2ed-b8d9-4853-b021-7691e292a1eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:22:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:06Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683e5f9034a57 namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: db33b8bf-3eb1-4b3c-8115-3c62ef056e19 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:22:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:06Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683e5fa521901 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: e77f2bd9-11e5-4963-9b6c-23c36df12bb2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:22:06Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:06Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683e5fa625660 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: 5abe2033-d0a3-43d0-9c39-053fa8befada reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:22:07Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 1.395s (1.395s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683e64d95017d namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 123ed22b-dae4-4b23-89b8-cf6a2439c5f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:22:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683e651ff2457 namespace: kserve-ci-e2e-test resourceVersion: "15841" uid: 42e6f725-6cd1-41ec-be72-02f365ab7209 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:22:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683e65262e654 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 4e6eff84-1cb7-465b-8079-930bbd7fa814 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T17:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:24:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:04Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c683e68ead89fa namespace: kserve-ci-e2e-test resourceVersion: "17268" uid: 3a4c7ca8-27a1-4ecd-853d-d62e39ce43a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:24:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:01Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c68400f440424c namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: a0185724-a3a8-403c-b39d-8ee1f4de5756 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:24:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:01Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c68400f440d88e namespace: kserve-ci-e2e-test resourceVersion: "17099" uid: 8f895a6b-949c-4f21-aede-2abdf67f9d9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 200856f8-0ef2-4cfe-ba47-e838dcd88ff0 kind: Event lastTimestamp: "2026-07-28T17:24:04Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:04Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc.18c684019116afcb namespace: kserve-ci-e2e-test resourceVersion: "17266" uid: 92d1bd9e-b7d2-4f92-9893-de9a01e029db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-84c58bd689 namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 16f0275d-9eae-4653-8b2d-3ae9c4a4e0e6 kind: Event lastTimestamp: "2026-07-28T17:21:31Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-84c58bd689-f96qc' metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:31Z" name: isvc-sklearn-graph-1-predictor-84c58bd689.18c683de054b7d91 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: 4e3c90c3-b98c-4180-bebd-e1b8124f591c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: cec13b7c-ad75-45c6-9f11-71711ba350a7 kind: Event lastTimestamp: "2026-07-28T17:21:31Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-84c58bd689 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:31Z" name: isvc-sklearn-graph-1-predictor.18c683de02684ab7 namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 5930ff71-6222-4372-81b5-0f42d0e4a1c3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 5cbc893e-9b13-46e5-94cd-cb3f0af302eb kind: Event lastTimestamp: "2026-07-28T17:22:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:46Z" name: isvc-sklearn-graph-1-predictor.18c683e18109385c namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 3679c096-069f-4327-a876-e4af5762b011 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 5cbc893e-9b13-46e5-94cd-cb3f0af302eb kind: Event lastTimestamp: "2026-07-28T17:22:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:46Z" name: isvc-sklearn-graph-1-predictor.18c683e18187da8b namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: 7d59b741-516b-46a4-b2d1-1fdc1cb35cc3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:23:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 5cbc893e-9b13-46e5-94cd-cb3f0af302eb kind: Event lastTimestamp: "2026-07-28T17:23:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:46Z" name: isvc-sklearn-graph-1-predictor.18c683f2fb9014b3 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: a5d57c7f-ed30-4dd0-92f0-86d2feead7de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:23:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 5cbc893e-9b13-46e5-94cd-cb3f0af302eb kind: Event lastTimestamp: "2026-07-28T17:23:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:46Z" name: isvc-sklearn-graph-1-predictor.18c683f2fc1923ef namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 92b92c3f-66e8-4ef7-8da9-42ad9d123345 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 56189d92-6604-4424-96d5-625634a4439b kind: Event lastTimestamp: "2026-07-28T17:23:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:24Z" name: isvc-sklearn-graph-1.18c683de025bd372 namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: b2f9055c-253a-4875-9e72-d1cfcb301687 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 56189d92-6604-4424-96d5-625634a4439b kind: Event lastTimestamp: "2026-07-28T17:23:24Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T17:23:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:24Z" name: isvc-sklearn-graph-1.18c683f8473d8c38 namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: 3f282040-706a-4ac2-bb0b-b706ff4ca938 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:21:32.305382Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683de1dd411a1 namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: a9262793-ef29-41c6-9db5-1153ec714da8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683de382a7334 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: ec5efb4b-cc52-464c-8496-f3b89dad334f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" already present on machine metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683de398e8901 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 52301795-1245-4ffc-80ed-2295fe3a2346 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683de3cad4eed namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: dbd49516-7438-46e5-ba20-6eb53b943ffa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683de3d20d864 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 9ac02e58-1235-4632-a854-c19f957ce776 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:21:37Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:21:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:37Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683df47946f77 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: c5bec308-54a5-47ec-a802-364598ada301 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:22:04Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" in 27.417s (27.417s including waiting). Image size: 1555722147 bytes.' metadata: creationTimestamp: "2026-07-28T17:22:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683e5a9c46f61 namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: 4be620d3-3de7-44fb-84ff-132a35a025fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:22:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:22:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683e5ae9fb5e7 namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 744f3b58-247c-4592-abea-e692eb43964f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:22:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:22:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683e5afb19f2a namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: df20779e-1b29-42a6-9a55-31243e25a0ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:22:04Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:22:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683e5afbd4e74 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: f4cd8935-333d-45b6-be95-57eacfe3b5ca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:22:07Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.648s (2.648s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:07Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683e64da0dfd8 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: a7a9cd97-a059-45ba-881e-a442d74f29c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:22:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:07Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683e6520626a4 namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: 31017348-7628-4d1e-af50-23862fbf93ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:22:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:07Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683e65274d96d namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 11d19325-f9ae-4d29-b565-f92aaeee1f0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T17:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:24:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c683e68e204f43 namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: d6564262-ca59-4a4c-b827-b6ce15a46370 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c68401019654df namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: 85a61dca-f12f-4465-b87f-cc471af2a5a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c684010196ef7f namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: cb6ee991-230c-4582-96af-0ee9987bda82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 52dbf1a2-5fec-4479-9b26-4f70ea29f129 kind: Event lastTimestamp: "2026-07-28T17:24:04Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:04Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6.18c68401911788e1 namespace: kserve-ci-e2e-test resourceVersion: "17267" uid: 3c19fca7-36f8-4a60-b5bf-219a86f9282d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-5bc9545b47 namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: ebb988a2-e263-423b-8ba1-c3b2b7785fe7 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-5bc9545b47-8h9k6' metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47.18c683de1d8659ad namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: 93714890-f604-410f-a2ab-f6a69bc117ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: 6c68df47-8f36-4a9b-8e7e-a5f24f684ac8 kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-5bc9545b47 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:32Z" name: isvc-sklearn-graph-2-predictor.18c683de1a508dc9 namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 13d3eca7-d10b-410e-8d13-fde536ebd998 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 406ec8f8-1c3e-494d-b3f3-4e8b1e58f446 kind: Event lastTimestamp: "2026-07-28T17:22:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:47Z" name: isvc-sklearn-graph-2-predictor.18c683e198f393f8 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: b3ed1772-9401-4b0c-a3bd-1ad97b160135 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 406ec8f8-1c3e-494d-b3f3-4e8b1e58f446 kind: Event lastTimestamp: "2026-07-28T17:22:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:47Z" name: isvc-sklearn-graph-2-predictor.18c683e199d7a54f namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 6ef6aef2-1605-4e98-ab79-ca20138f3ef9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 406ec8f8-1c3e-494d-b3f3-4e8b1e58f446 kind: Event lastTimestamp: "2026-07-28T17:23:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:47Z" name: isvc-sklearn-graph-2-predictor.18c683f3141ce2df namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: 28008381-3d43-405c-aa4a-f6e79e7510d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 406ec8f8-1c3e-494d-b3f3-4e8b1e58f446 kind: Event lastTimestamp: "2026-07-28T17:23:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:47Z" name: isvc-sklearn-graph-2-predictor.18c683f3149d6abb namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: 25fecbb6-1f4b-47ab-97f9-05d04183ff2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 1151a4d1-a70b-4313-bc15-d7c146813759 kind: Event lastTimestamp: "2026-07-28T17:23:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:24Z" name: isvc-sklearn-graph-2.18c683de1a6230f4 namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: 277dbad2-f61c-4d26-8b4d-c832183a4988 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: 1151a4d1-a70b-4313-bc15-d7c146813759 kind: Event lastTimestamp: "2026-07-28T17:23:24Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T17:23:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:24Z" name: isvc-sklearn-graph-2.18c683f84a64507b namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: 7ec03289-2d3b-4be5-814a-c86ead3a403d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:21:32.020729Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-v4mmp to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:21:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683de0cdc7dc0 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 676d30c5-6266-43bd-ae58-685527dc9fe8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:21:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683de262faf26 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 1a07c46f-d0c7-4bed-9b19-1632317d07be reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3049312fa55515cbc94d5484b2c3ba6994735bde07ec862ad9b359b89e6c0af0" already present on machine metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683de27a96c4a namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: 8905e4ca-97b3-41a5-988f-b3f95f315cc6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683de2a16b23a namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: 276f5ce2-0566-4982-9e79-a2afe1d23a1d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683de2a88a984 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: d23a8dc1-787f-4a02-8399-b2848105a006 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:21:37Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T17:21:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683df47b1b343 namespace: kserve-ci-e2e-test resourceVersion: "15547" uid: 428980da-24de-45f9-9db1-c70aee5137a6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:22:05Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.849s (27.849s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T17:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683e5c3a9dfff namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: 7eb0e3f1-29e0-4815-9148-352bfa576041 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:22:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683e5c9203a29 namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: 67b405ea-dd43-449a-99ab-7208a9200d0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:22:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683e5ca1d1d76 namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 27b9d72b-4443-421f-be86-5003e5741438 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:22:05Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683e5ca2b9704 namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: 630448a0-8b56-49ee-8077-b17b51887bf8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:22:07Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.209s (2.209s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683e64de4c953 namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 419509d1-ecca-4bd7-a929-1ee3e25f3882 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:22:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683e651fdd51f namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: da548a0c-e2d2-41e0-b27d-16ee78e4f094 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:22:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683e6526a1442 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: 27173499-07ca-4282-825c-853dc5c2e423 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:24:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c683e68e66ff4d namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: ffdaf810-acff-4f95-b9b3-c59a2e851cd9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:24:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c68400f79901fd namespace: kserve-ci-e2e-test resourceVersion: "17124" uid: 1a1941de-b7a3-4189-8a77-99ab868e9ffe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:24:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c68400f799738d namespace: kserve-ci-e2e-test resourceVersion: "17126" uid: 2f6b011a-a4f1-465d-b4ad-bfa41e7f4ab3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 634e2466-37ed-4256-9178-dd70bcc947ba kind: Event lastTimestamp: "2026-07-28T17:24:05Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-v4mmp.18c68401ccfc07f1 namespace: kserve-ci-e2e-test resourceVersion: "17278" uid: 9d74cd6a-e2ff-49cc-96cb-731afc1c4032 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: f2de0533-f375-459c-a430-6ac1ae3ef72d kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-v4mmp' metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:32Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c683de0c75f46a namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: f5d5800b-99b9-4962-ac9b-ec197fbc5b73 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: 628c5744-23a4-4597-bed4-06f27501de16 kind: Event lastTimestamp: "2026-07-28T17:21:31Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:31Z" name: isvc-xgboost-graph-predictor.18c683de099cc0b0 namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: 7148581a-ccf0-4980-945b-e073415f7bae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 65986de6-7bbc-40da-848f-5d77db2b4b06 kind: Event lastTimestamp: "2026-07-28T17:22:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:47Z" name: isvc-xgboost-graph-predictor.18c683e18858692b namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: ab929a6d-a69b-4b3a-81be-fc1bc0b65b44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 65986de6-7bbc-40da-848f-5d77db2b4b06 kind: Event lastTimestamp: "2026-07-28T17:22:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:47Z" name: isvc-xgboost-graph-predictor.18c683e188dfbd62 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: b66c6d16-1499-4519-9349-09a81ccdf18a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 65986de6-7bbc-40da-848f-5d77db2b4b06 kind: Event lastTimestamp: "2026-07-28T17:23:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:32Z" name: isvc-xgboost-graph-predictor.18c683f3038abf93 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 20ec023b-e6f9-4f90-bb8c-922ad9cd8a13 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 65986de6-7bbc-40da-848f-5d77db2b4b06 kind: Event lastTimestamp: "2026-07-28T17:23:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:32Z" name: isvc-xgboost-graph-predictor.18c683f30429e2ca namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: bb721e5e-820c-458a-96b6-9de9d90f75a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: b7a5144d-cc8e-431f-b90b-7380868d17ff kind: Event lastTimestamp: "2026-07-28T17:23:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:05Z" name: isvc-xgboost-graph.18c683de099b7797 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: ab199f1e-b5d5-4be7-8b6c-f9e6523c683b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: b7a5144d-cc8e-431f-b90b-7380868d17ff kind: Event lastTimestamp: "2026-07-28T17:23:05Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T17:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:05Z" name: isvc-xgboost-graph.18c683f3daaace4d namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: 418cc638-8abf-4763-9aa8-2c146187be07 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:23:51.662246Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64698579b9-xlx8n namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: f01437bd-789e-4b2f-a441-7c7fd3c7e0ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-64698579b9-xlx8n to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:23:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:23:51Z" name: model-chainer-64698579b9-xlx8n.18c683fe902506eb namespace: kserve-ci-e2e-test resourceVersion: "16961" uid: 92c56ded-8f2e-41e2-bd55-e73a79acc76e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:51Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64698579b9-xlx8n namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: f01437bd-789e-4b2f-a441-7c7fd3c7e0ce kind: Event lastTimestamp: "2026-07-28T17:23:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:51Z" name: model-chainer-64698579b9-xlx8n.18c683fe979d1019 namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: 4df3cc27-badc-4e43-a0aa-b19792dc1bc1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:52Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-64698579b9-xlx8n namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: f01437bd-789e-4b2f-a441-7c7fd3c7e0ce kind: Event lastTimestamp: "2026-07-28T17:23:52Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:23:52Z" name: model-chainer-64698579b9-xlx8n.18c683fecd2d4217 namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: 4570a446-780b-4b98-9632-98a682d6b521 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-xlx8n namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: f01437bd-789e-4b2f-a441-7c7fd3c7e0ce kind: Event lastTimestamp: "2026-07-28T17:23:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:52Z" name: model-chainer-64698579b9-xlx8n.18c683fece839796 namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: d99228c7-94f7-445b-a489-23f77d2b1139 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-xlx8n namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: f01437bd-789e-4b2f-a441-7c7fd3c7e0ce kind: Event lastTimestamp: "2026-07-28T17:23:52Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T17:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:52Z" name: model-chainer-64698579b9-xlx8n.18c683fed1c1ac6d namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: 317cc5c8-efc9-4e65-a549-7deaeaa33771 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-xlx8n namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: f01437bd-789e-4b2f-a441-7c7fd3c7e0ce kind: Event lastTimestamp: "2026-07-28T17:23:52Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T17:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:52Z" name: model-chainer-64698579b9-xlx8n.18c683fed238771e namespace: kserve-ci-e2e-test resourceVersion: "16985" uid: dad645b4-6b81-4e62-bce7-20c52858582d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-xlx8n namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: f01437bd-789e-4b2f-a441-7c7fd3c7e0ce kind: Event lastTimestamp: "2026-07-28T17:24:01Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T17:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:01Z" name: model-chainer-64698579b9-xlx8n.18c68400eada0758 namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: de690a2a-f46c-4af0-93da-5f44bd5366c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-64698579b9-xlx8n namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: f01437bd-789e-4b2f-a441-7c7fd3c7e0ce kind: Event lastTimestamp: "2026-07-28T17:24:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:29Z" name: model-chainer-64698579b9-xlx8n.18c684019d02ef61 namespace: kserve-ci-e2e-test resourceVersion: "17484" uid: 38cfab1e-a93d-47cb-bc32-2db754cfe9dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-64698579b9 namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: 54bfdc94-8078-42ae-af28-c9573a917826 kind: Event lastTimestamp: "2026-07-28T17:23:51Z" message: 'Created pod: model-chainer-64698579b9-xlx8n' metadata: creationTimestamp: "2026-07-28T17:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:51Z" name: model-chainer-64698579b9.18c683fe8f9c5b84 namespace: kserve-ci-e2e-test resourceVersion: "16957" uid: 66a394e4-9cac-46ba-a3c2-98d967f11fe1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16950" uid: f7922f09-f8d4-475e-a797-67621345147b kind: Event lastTimestamp: "2026-07-28T17:23:51Z" message: Scaled up replica set model-chainer-64698579b9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:51Z" name: model-chainer.18c683fe8ec5b9af namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: 0d35ee97-dea0-4124-a988-8132a423cfb7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16967" uid: b451626d-d936-43d4-bc52-e03511e01687 kind: Event lastTimestamp: "2026-07-28T17:23:51Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T17:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:51Z" name: model-chainer.18c683fe936e6056 namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: bf8e5fe9-cc9e-4f5d-be69-5b00851b6191 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:25:02.001769Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-211cd-755c5c64d7-dsdg5 namespace: kserve-ci-e2e-test resourceVersion: "17756" uid: ceb64d7b-d69f-4fe6-a4df-a2dc2a97b5a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-211cd-755c5c64d7-dsdg5 to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:25:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:25:02Z" name: sequence-graph-211cd-755c5c64d7-dsdg5.18c6840ef0b4e8a0 namespace: kserve-ci-e2e-test resourceVersion: "17761" uid: 2ddaa3de-a2d2-43e5-8375-f5b0f334ca11 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:25:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-211cd-755c5c64d7-dsdg5 namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: ceb64d7b-d69f-4fe6-a4df-a2dc2a97b5a4 kind: Event lastTimestamp: "2026-07-28T17:25:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-211cd-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:25:02Z" name: sequence-graph-211cd-755c5c64d7-dsdg5.18c6840efa3c0128 namespace: kserve-ci-e2e-test resourceVersion: "17777" uid: 1056ff37-341f-4af2-8f71-e200e9ea45e7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:25:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-211cd-755c5c64d7-dsdg5 namespace: kserve-ci-e2e-test resourceVersion: "17765" uid: ceb64d7b-d69f-4fe6-a4df-a2dc2a97b5a4 kind: Event lastTimestamp: "2026-07-28T17:25:03Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:25:03Z" name: sequence-graph-211cd-755c5c64d7-dsdg5.18c6840f2de6af77 namespace: kserve-ci-e2e-test resourceVersion: "17787" uid: 41fc078a-6f38-4cdd-9fac-a52fd8062e05 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-211cd} kind: Pod name: sequence-graph-211cd-755c5c64d7-dsdg5 namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: ceb64d7b-d69f-4fe6-a4df-a2dc2a97b5a4 kind: Event lastTimestamp: "2026-07-28T17:25:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:25:03Z" name: sequence-graph-211cd-755c5c64d7-dsdg5.18c6840f2f4f4292 namespace: kserve-ci-e2e-test resourceVersion: "17789" uid: ec1184c7-1cd9-4788-8ae5-981ecbc27a21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-211cd} kind: Pod name: sequence-graph-211cd-755c5c64d7-dsdg5 namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: ceb64d7b-d69f-4fe6-a4df-a2dc2a97b5a4 kind: Event lastTimestamp: "2026-07-28T17:25:03Z" message: 'Created container: sequence-graph-211cd' metadata: creationTimestamp: "2026-07-28T17:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:25:03Z" name: sequence-graph-211cd-755c5c64d7-dsdg5.18c6840f32690a75 namespace: kserve-ci-e2e-test resourceVersion: "17790" uid: 8eb5557f-0758-4862-9ace-1e72768ff082 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-211cd} kind: Pod name: sequence-graph-211cd-755c5c64d7-dsdg5 namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: ceb64d7b-d69f-4fe6-a4df-a2dc2a97b5a4 kind: Event lastTimestamp: "2026-07-28T17:25:03Z" message: Started container sequence-graph-211cd metadata: creationTimestamp: "2026-07-28T17:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:25:03Z" name: sequence-graph-211cd-755c5c64d7-dsdg5.18c6840f32e0ebef namespace: kserve-ci-e2e-test resourceVersion: "17791" uid: 5c8e5c67-7817-4964-909c-0201e8d5137e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-211cd} kind: Pod name: sequence-graph-211cd-755c5c64d7-dsdg5 namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: ceb64d7b-d69f-4fe6-a4df-a2dc2a97b5a4 kind: Event lastTimestamp: "2026-07-28T17:33:16Z" message: Stopping container sequence-graph-211cd metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:16Z" name: sequence-graph-211cd-755c5c64d7-dsdg5.18c684821de3063a namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 7cd4a525-793c-4905-b0d7-a188b6665a8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-211cd} kind: Pod name: sequence-graph-211cd-755c5c64d7-dsdg5 namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: ceb64d7b-d69f-4fe6-a4df-a2dc2a97b5a4 kind: Event lastTimestamp: "2026-07-28T17:33:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:33:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:45Z" name: sequence-graph-211cd-755c5c64d7-dsdg5.18c68482e51f64f4 namespace: kserve-ci-e2e-test resourceVersion: "21942" uid: a00d3ef5-784f-4f6f-80e5-cb1f209a0617 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:25:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-211cd-755c5c64d7 namespace: kserve-ci-e2e-test resourceVersion: "17753" uid: 16166f86-81cf-4ae0-9b65-ad66ca0cec26 kind: Event lastTimestamp: "2026-07-28T17:25:01Z" message: 'Created pod: sequence-graph-211cd-755c5c64d7-dsdg5' metadata: creationTimestamp: "2026-07-28T17:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:25:01Z" name: sequence-graph-211cd-755c5c64d7.18c6840ef03af073 namespace: kserve-ci-e2e-test resourceVersion: "17757" uid: e123bccc-961f-4d21-844d-83e9d85df39e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:25:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-211cd namespace: kserve-ci-e2e-test resourceVersion: "17751" uid: f318b076-1afb-4bbd-9f1d-67e6663a92fe kind: Event lastTimestamp: "2026-07-28T17:25:01Z" message: Scaled up replica set sequence-graph-211cd-755c5c64d7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:25:01Z" name: sequence-graph-211cd.18c6840eef643620 namespace: kserve-ci-e2e-test resourceVersion: "17754" uid: bc73b04e-c837-4d76-87c8-b67fa175e508 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:25:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-211cd namespace: kserve-ci-e2e-test resourceVersion: "17769" uid: 5adcd25f-f289-4351-bfe6-b8a204efcf12 kind: Event lastTimestamp: "2026-07-28T17:25:02Z" message: InferenceGraph [sequence-graph-211cd] is Ready metadata: creationTimestamp: "2026-07-28T17:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:25:02Z" name: sequence-graph-211cd.18c6840ef4beb479 namespace: kserve-ci-e2e-test resourceVersion: "17771" uid: 4c420b02-5127-41c9-ac14-164c4d3531d1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:25:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-211cd namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: 13b56deb-f208-4384-8efd-450c3886be3a kind: Event lastTimestamp: "2026-07-28T17:25:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:25:47Z" name: sequence-graph-211cd.18c684126e810225 namespace: kserve-ci-e2e-test resourceVersion: "18108" uid: 9e325bdb-e0b2-4e05-847c-96083e89f2bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:25:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-211cd namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: 13b56deb-f208-4384-8efd-450c3886be3a kind: Event lastTimestamp: "2026-07-28T17:25:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:25:47Z" name: sequence-graph-211cd.18c684126eee60d6 namespace: kserve-ci-e2e-test resourceVersion: "18109" uid: c3309175-130e-4a22-ba0d-2a219890dccb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:35:27.321603Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a513a-76cfc9c946-chvjf namespace: kserve-ci-e2e-test resourceVersion: "23032" uid: 0d52c5df-884d-4976-b689-b7bf15b73269 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a513a-76cfc9c946-chvjf to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:35:27Z" name: sequence-graph-a513a-76cfc9c946-chvjf.18c684a088ac68ec namespace: kserve-ci-e2e-test resourceVersion: "23038" uid: 34c7e8e7-19ac-466b-9c44-76113909bddb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a513a-76cfc9c946-chvjf namespace: kserve-ci-e2e-test resourceVersion: "23041" uid: 0d52c5df-884d-4976-b689-b7bf15b73269 kind: Event lastTimestamp: "2026-07-28T17:35:27Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:35:27Z" name: sequence-graph-a513a-76cfc9c946-chvjf.18c684a0a21ffe5d namespace: kserve-ci-e2e-test resourceVersion: "23054" uid: 39608c49-f61e-446b-a52f-0a5f52acad21 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a513a} kind: Pod name: sequence-graph-a513a-76cfc9c946-chvjf namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: 0d52c5df-884d-4976-b689-b7bf15b73269 kind: Event lastTimestamp: "2026-07-28T17:35:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:27Z" name: sequence-graph-a513a-76cfc9c946-chvjf.18c684a0a39f7d08 namespace: kserve-ci-e2e-test resourceVersion: "23056" uid: 8524dc5a-85f3-4270-8f21-be16496a367d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a513a} kind: Pod name: sequence-graph-a513a-76cfc9c946-chvjf namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: 0d52c5df-884d-4976-b689-b7bf15b73269 kind: Event lastTimestamp: "2026-07-28T17:35:27Z" message: 'Created container: sequence-graph-a513a' metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:27Z" name: sequence-graph-a513a-76cfc9c946-chvjf.18c684a0a6935fcc namespace: kserve-ci-e2e-test resourceVersion: "23057" uid: bbb2ad6f-1748-4f35-9101-14ff3bd92aa3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a513a} kind: Pod name: sequence-graph-a513a-76cfc9c946-chvjf namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: 0d52c5df-884d-4976-b689-b7bf15b73269 kind: Event lastTimestamp: "2026-07-28T17:35:27Z" message: Started container sequence-graph-a513a metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:27Z" name: sequence-graph-a513a-76cfc9c946-chvjf.18c684a0a70759c8 namespace: kserve-ci-e2e-test resourceVersion: "23058" uid: bbcd3833-52b0-4948-8be4-e2463e35a8ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a513a} kind: Pod name: sequence-graph-a513a-76cfc9c946-chvjf namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: 0d52c5df-884d-4976-b689-b7bf15b73269 kind: Event lastTimestamp: "2026-07-28T17:43:41Z" message: Stopping container sequence-graph-a513a metadata: creationTimestamp: "2026-07-28T17:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:41Z" name: sequence-graph-a513a-76cfc9c946-chvjf.18c68513b3a318d0 namespace: kserve-ci-e2e-test resourceVersion: "26972" uid: 05329099-ac73-45ad-bf05-d7785eb33bff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a513a} kind: Pod name: sequence-graph-a513a-76cfc9c946-chvjf namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: 0d52c5df-884d-4976-b689-b7bf15b73269 kind: Event lastTimestamp: "2026-07-28T17:44:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:09Z" name: sequence-graph-a513a-76cfc9c946-chvjf.18c685145f25791e namespace: kserve-ci-e2e-test resourceVersion: "27518" uid: c0327c37-9087-47b4-b08e-b008210e1d53 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a513a-76cfc9c946 namespace: kserve-ci-e2e-test resourceVersion: "23028" uid: d0ebde38-28f2-42a1-b7e2-d7566a409bd6 kind: Event lastTimestamp: "2026-07-28T17:35:27Z" message: 'Created pod: sequence-graph-a513a-76cfc9c946-chvjf' metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:27Z" name: sequence-graph-a513a-76cfc9c946.18c684a0883623c3 namespace: kserve-ci-e2e-test resourceVersion: "23033" uid: 9ea0a9b6-314f-41c0-891a-96cb3d104fe3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a513a namespace: kserve-ci-e2e-test resourceVersion: "23025" uid: 6614a651-89f5-4117-9d36-d877b99f6b85 kind: Event lastTimestamp: "2026-07-28T17:35:27Z" message: Scaled up replica set sequence-graph-a513a-76cfc9c946 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:27Z" name: sequence-graph-a513a.18c684a087759b43 namespace: kserve-ci-e2e-test resourceVersion: "23030" uid: 0e1ec158-e289-43fd-91b0-c3e06dda2450 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a513a namespace: kserve-ci-e2e-test resourceVersion: "23043" uid: a71f5e61-a9b5-4bd9-b5d3-de3cea10ed02 kind: Event lastTimestamp: "2026-07-28T17:35:27Z" message: InferenceGraph [sequence-graph-a513a] is Ready metadata: creationTimestamp: "2026-07-28T17:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:27Z" name: sequence-graph-a513a.18c684a08cedadce namespace: kserve-ci-e2e-test resourceVersion: "23046" uid: aae506b8-b4d6-4bca-8516-5e819848a66e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a513a namespace: kserve-ci-e2e-test resourceVersion: "23027" uid: 831cbabb-edf3-4c45-928d-a418ee684386 kind: Event lastTimestamp: "2026-07-28T17:35:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:42Z" name: sequence-graph-a513a.18c684a405dfb95f namespace: kserve-ci-e2e-test resourceVersion: "23168" uid: 7cc8e156-f561-4da4-83c9-a5ddbb12f65a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a513a namespace: kserve-ci-e2e-test resourceVersion: "23027" uid: 831cbabb-edf3-4c45-928d-a418ee684386 kind: Event lastTimestamp: "2026-07-28T17:35:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:42Z" name: sequence-graph-a513a.18c684a406423dc9 namespace: kserve-ci-e2e-test resourceVersion: "23170" uid: 2381853f-e415-4bba-b255-0055c5caf646 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a513a namespace: kserve-ci-e2e-test resourceVersion: "23169" uid: 831cbabb-edf3-4c45-928d-a418ee684386 kind: Event lastTimestamp: "2026-07-28T17:35:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:57Z" name: sequence-graph-a513a.18c684a785039ccc namespace: kserve-ci-e2e-test resourceVersion: "23280" uid: 58b8f408-9baa-41c5-94ce-94ce584b9158 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a513a namespace: kserve-ci-e2e-test resourceVersion: "23169" uid: 831cbabb-edf3-4c45-928d-a418ee684386 kind: Event lastTimestamp: "2026-07-28T17:35:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:57Z" name: sequence-graph-a513a.18c684a785703f10 namespace: kserve-ci-e2e-test resourceVersion: "23282" uid: 9dad37e7-40bb-44b6-8f36-6e780f76b8e8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:34:16.954356Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ef873-59745d8759-5ksgm namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 8b2806b1-7f88-4b15-84c9-e2c422c0e049 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-ef873-59745d8759-5ksgm to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:34:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:34:16Z" name: sequence-graph-ef873-59745d8759-5ksgm.18c6849026756ab4 namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 7958ed3b-c58a-4015-a940-ab90a5501db7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:17Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ef873-59745d8759-5ksgm namespace: kserve-ci-e2e-test resourceVersion: "22234" uid: 8b2806b1-7f88-4b15-84c9-e2c422c0e049 kind: Event lastTimestamp: "2026-07-28T17:34:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-ef873-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:17Z" name: sequence-graph-ef873-59745d8759-5ksgm.18c68490329b00c8 namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: 7cef8153-37b3-4f72-8140-afbe8d600f69 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:17Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ef873-59745d8759-5ksgm namespace: kserve-ci-e2e-test resourceVersion: "22242" uid: 8b2806b1-7f88-4b15-84c9-e2c422c0e049 kind: Event lastTimestamp: "2026-07-28T17:34:17Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:34:17Z" name: sequence-graph-ef873-59745d8759-5ksgm.18c6849063a1be4e namespace: kserve-ci-e2e-test resourceVersion: "22260" uid: 99f03a61-b0af-4eb0-9307-2f39e1438b72 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ef873} kind: Pod name: sequence-graph-ef873-59745d8759-5ksgm namespace: kserve-ci-e2e-test resourceVersion: "22234" uid: 8b2806b1-7f88-4b15-84c9-e2c422c0e049 kind: Event lastTimestamp: "2026-07-28T17:34:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:18Z" name: sequence-graph-ef873-59745d8759-5ksgm.18c6849064f82bbf namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: fdf0b0ad-0597-4ae0-a759-1fac1600cec7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ef873} kind: Pod name: sequence-graph-ef873-59745d8759-5ksgm namespace: kserve-ci-e2e-test resourceVersion: "22234" uid: 8b2806b1-7f88-4b15-84c9-e2c422c0e049 kind: Event lastTimestamp: "2026-07-28T17:34:18Z" message: 'Created container: sequence-graph-ef873' metadata: creationTimestamp: "2026-07-28T17:34:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:18Z" name: sequence-graph-ef873-59745d8759-5ksgm.18c6849067e546eb namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: e3c88af2-8ca2-449e-8fce-b0bfd5fcc723 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ef873} kind: Pod name: sequence-graph-ef873-59745d8759-5ksgm namespace: kserve-ci-e2e-test resourceVersion: "22234" uid: 8b2806b1-7f88-4b15-84c9-e2c422c0e049 kind: Event lastTimestamp: "2026-07-28T17:34:18Z" message: Started container sequence-graph-ef873 metadata: creationTimestamp: "2026-07-28T17:34:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:18Z" name: sequence-graph-ef873-59745d8759-5ksgm.18c684906868b1b0 namespace: kserve-ci-e2e-test resourceVersion: "22265" uid: c8d24292-2709-46d6-b9a0-8ed44cf8a6cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ef873} kind: Pod name: sequence-graph-ef873-59745d8759-5ksgm namespace: kserve-ci-e2e-test resourceVersion: "22234" uid: 8b2806b1-7f88-4b15-84c9-e2c422c0e049 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Stopping container sequence-graph-ef873 metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: sequence-graph-ef873-59745d8759-5ksgm.18c684927f8df029 namespace: kserve-ci-e2e-test resourceVersion: "22363" uid: 4f96d04f-0cf5-4b98-88b4-ef5ee46fefc7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ef873} kind: Pod name: sequence-graph-ef873-59745d8759-5ksgm namespace: kserve-ci-e2e-test resourceVersion: "22234" uid: 8b2806b1-7f88-4b15-84c9-e2c422c0e049 kind: Event lastTimestamp: "2026-07-28T17:34:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:54Z" name: sequence-graph-ef873-59745d8759-5ksgm.18c6849314f2c105 namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 9ea7fb45-b0f4-4746-ad1d-d23333a0b071 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-ef873-59745d8759 namespace: kserve-ci-e2e-test resourceVersion: "22228" uid: 3a331c57-4ed0-4d3e-9d0d-f788253aceab kind: Event lastTimestamp: "2026-07-28T17:34:16Z" message: 'Created pod: sequence-graph-ef873-59745d8759-5ksgm' metadata: creationTimestamp: "2026-07-28T17:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:16Z" name: sequence-graph-ef873-59745d8759.18c68490260c8816 namespace: kserve-ci-e2e-test resourceVersion: "22233" uid: 227b3a35-2d1d-4a1a-a92f-48cd774f74d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-ef873 namespace: kserve-ci-e2e-test resourceVersion: "22226" uid: ddf1dcd7-456a-4018-b798-f74751feb9a9 kind: Event lastTimestamp: "2026-07-28T17:34:16Z" message: Scaled up replica set sequence-graph-ef873-59745d8759 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:16Z" name: sequence-graph-ef873.18c6849025361786 namespace: kserve-ci-e2e-test resourceVersion: "22230" uid: 5aaf1703-0cbc-441b-b736-856192c6e646 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ef873 namespace: kserve-ci-e2e-test resourceVersion: "22244" uid: 0ffb46ce-2187-4cb9-ac43-2f9ae42e2945 kind: Event lastTimestamp: "2026-07-28T17:34:17Z" message: InferenceGraph [sequence-graph-ef873] is Ready metadata: creationTimestamp: "2026-07-28T17:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:17Z" name: sequence-graph-ef873.18c684902b8f7b88 namespace: kserve-ci-e2e-test resourceVersion: "22246" uid: 9cdd9994-1e10-424a-95a8-715b6ca227c5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:46.312278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-11994-677669c5c7-fdfzq namespace: kserve-ci-e2e-test resourceVersion: "27151" uid: 30a5328e-bc85-487d-b33e-24d1b95856b5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-11994-677669c5c7-fdfzq to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:46Z" name: splitter-graph-11994-677669c5c7-fdfzq.18c68514b6d5d851 namespace: kserve-ci-e2e-test resourceVersion: "27157" uid: 731687dd-58d9-4e03-841a-b3620a40d4c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:46Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-11994-677669c5c7-fdfzq namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 30a5328e-bc85-487d-b33e-24d1b95856b5 kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-11994-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:46Z" name: splitter-graph-11994-677669c5c7-fdfzq.18c68514c4950a7e namespace: kserve-ci-e2e-test resourceVersion: "27173" uid: f5f6ce12-0ca3-4aa7-b9ab-c26a886c8c17 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:47Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-11994-677669c5c7-fdfzq namespace: kserve-ci-e2e-test resourceVersion: "27162" uid: 30a5328e-bc85-487d-b33e-24d1b95856b5 kind: Event lastTimestamp: "2026-07-28T17:43:47Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:47Z" name: splitter-graph-11994-677669c5c7-fdfzq.18c68514f45a2226 namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: b738970f-4235-47f9-9acc-383a73e7667d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11994} kind: Pod name: splitter-graph-11994-677669c5c7-fdfzq namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 30a5328e-bc85-487d-b33e-24d1b95856b5 kind: Event lastTimestamp: "2026-07-28T17:43:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:47Z" name: splitter-graph-11994-677669c5c7-fdfzq.18c68514f591e3d5 namespace: kserve-ci-e2e-test resourceVersion: "27184" uid: be9be9ce-0b85-46ff-a880-c57305a2eb71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11994} kind: Pod name: splitter-graph-11994-677669c5c7-fdfzq namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 30a5328e-bc85-487d-b33e-24d1b95856b5 kind: Event lastTimestamp: "2026-07-28T17:43:47Z" message: 'Created container: splitter-graph-11994' metadata: creationTimestamp: "2026-07-28T17:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:47Z" name: splitter-graph-11994-677669c5c7-fdfzq.18c68514f8c86591 namespace: kserve-ci-e2e-test resourceVersion: "27185" uid: c552b278-165c-491c-8e84-20d6aa6dcc42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11994} kind: Pod name: splitter-graph-11994-677669c5c7-fdfzq namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 30a5328e-bc85-487d-b33e-24d1b95856b5 kind: Event lastTimestamp: "2026-07-28T17:43:47Z" message: Started container splitter-graph-11994 metadata: creationTimestamp: "2026-07-28T17:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:47Z" name: splitter-graph-11994-677669c5c7-fdfzq.18c68514f940fbb7 namespace: kserve-ci-e2e-test resourceVersion: "27186" uid: 1c5d702b-50dc-4b47-a9ea-7d9f00d2ef90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11994} kind: Pod name: splitter-graph-11994-677669c5c7-fdfzq namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 30a5328e-bc85-487d-b33e-24d1b95856b5 kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: Stopping container splitter-graph-11994 metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:56Z" name: splitter-graph-11994-677669c5c7-fdfzq.18c685170e9ff060 namespace: kserve-ci-e2e-test resourceVersion: "27274" uid: e4b31e34-2e9d-41c6-b5d0-5cb1dd401aef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11994} kind: Pod name: splitter-graph-11994-677669c5c7-fdfzq namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 30a5328e-bc85-487d-b33e-24d1b95856b5 kind: Event lastTimestamp: "2026-07-28T17:44:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:23Z" name: splitter-graph-11994-677669c5c7-fdfzq.18c68517a107d80b namespace: kserve-ci-e2e-test resourceVersion: "27631" uid: 04291ed3-28ee-49b5-8aad-e256e7e285aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-11994-677669c5c7 namespace: kserve-ci-e2e-test resourceVersion: "27148" uid: 569e3922-2063-4415-b419-49619b1f70de kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: 'Created pod: splitter-graph-11994-677669c5c7-fdfzq' metadata: creationTimestamp: "2026-07-28T17:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:46Z" name: splitter-graph-11994-677669c5c7.18c68514b6663ff8 namespace: kserve-ci-e2e-test resourceVersion: "27153" uid: 634a7c0d-bf32-4d9b-9deb-bbbb67879821 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-11994 namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: 63edf9b2-77e2-468f-b56d-b28d53926db9 kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: Scaled up replica set splitter-graph-11994-677669c5c7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:46Z" name: splitter-graph-11994.18c68514b5708507 namespace: kserve-ci-e2e-test resourceVersion: "27149" uid: 406803c0-0478-432f-8fb1-3d0ae93e7f15 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-11994 namespace: kserve-ci-e2e-test resourceVersion: "27167" uid: 06e72eae-3aff-48df-96ab-07ab07d57104 kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: InferenceGraph [splitter-graph-11994] is Ready metadata: creationTimestamp: "2026-07-28T17:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:46Z" name: splitter-graph-11994.18c68514bc0f3e37 namespace: kserve-ci-e2e-test resourceVersion: "27169" uid: 9bdfabd8-f7cf-4abf-ac99-153aa04973e2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:44:56.641382Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-824fb-7b8cd8fdfd-pdxxk namespace: kserve-ci-e2e-test resourceVersion: "27974" uid: d7e5e47e-7df9-4193-bb5d-7a4aecca45ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-824fb-7b8cd8fdfd-pdxxk to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:44:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:44:56Z" name: splitter-graph-824fb-7b8cd8fdfd-pdxxk.18c6852516c6dcd5 namespace: kserve-ci-e2e-test resourceVersion: "27977" uid: 43e10173-ad05-4a9d-8b05-ffa22c23d2ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:57Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-824fb-7b8cd8fdfd-pdxxk namespace: kserve-ci-e2e-test resourceVersion: "27983" uid: d7e5e47e-7df9-4193-bb5d-7a4aecca45ce kind: Event lastTimestamp: "2026-07-28T17:44:57Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:44:57Z" name: splitter-graph-824fb-7b8cd8fdfd-pdxxk.18c685252ffeff44 namespace: kserve-ci-e2e-test resourceVersion: "27996" uid: 67002d10-d6ec-4651-99d3-7a7b2b94701d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-824fb} kind: Pod name: splitter-graph-824fb-7b8cd8fdfd-pdxxk namespace: kserve-ci-e2e-test resourceVersion: "27976" uid: d7e5e47e-7df9-4193-bb5d-7a4aecca45ce kind: Event lastTimestamp: "2026-07-28T17:44:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:57Z" name: splitter-graph-824fb-7b8cd8fdfd-pdxxk.18c685253168ae99 namespace: kserve-ci-e2e-test resourceVersion: "27998" uid: a270dadf-5d8a-4838-a894-51f701c8fe7f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-824fb} kind: Pod name: splitter-graph-824fb-7b8cd8fdfd-pdxxk namespace: kserve-ci-e2e-test resourceVersion: "27976" uid: d7e5e47e-7df9-4193-bb5d-7a4aecca45ce kind: Event lastTimestamp: "2026-07-28T17:44:57Z" message: 'Created container: splitter-graph-824fb' metadata: creationTimestamp: "2026-07-28T17:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:57Z" name: splitter-graph-824fb-7b8cd8fdfd-pdxxk.18c685253452b491 namespace: kserve-ci-e2e-test resourceVersion: "28000" uid: 6e857aff-c6ba-43f1-bf02-9cbd54d39c1f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-824fb} kind: Pod name: splitter-graph-824fb-7b8cd8fdfd-pdxxk namespace: kserve-ci-e2e-test resourceVersion: "27976" uid: d7e5e47e-7df9-4193-bb5d-7a4aecca45ce kind: Event lastTimestamp: "2026-07-28T17:44:57Z" message: Started container splitter-graph-824fb metadata: creationTimestamp: "2026-07-28T17:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:57Z" name: splitter-graph-824fb-7b8cd8fdfd-pdxxk.18c6852534c07572 namespace: kserve-ci-e2e-test resourceVersion: "28001" uid: 5cc76acf-7112-4ef0-a520-9c99fbf6708d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-824fb} kind: Pod name: splitter-graph-824fb-7b8cd8fdfd-pdxxk namespace: kserve-ci-e2e-test resourceVersion: "27976" uid: d7e5e47e-7df9-4193-bb5d-7a4aecca45ce kind: Event lastTimestamp: "2026-07-28T17:53:11Z" message: Stopping container splitter-graph-824fb metadata: creationTimestamp: "2026-07-28T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:11Z" name: splitter-graph-824fb-7b8cd8fdfd-pdxxk.18c685983b5d36cb namespace: kserve-ci-e2e-test resourceVersion: "31450" uid: 59ed9d41-7ed0-48f1-8a92-c5fda73e8250 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:53:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-824fb} kind: Pod name: splitter-graph-824fb-7b8cd8fdfd-pdxxk namespace: kserve-ci-e2e-test resourceVersion: "27976" uid: d7e5e47e-7df9-4193-bb5d-7a4aecca45ce kind: Event lastTimestamp: "2026-07-28T17:53:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:53:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:39Z" name: splitter-graph-824fb-7b8cd8fdfd-pdxxk.18c68598eb1a0000 namespace: kserve-ci-e2e-test resourceVersion: "31686" uid: d526c143-41db-4917-98a0-31babc82c6ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-824fb-7b8cd8fdfd namespace: kserve-ci-e2e-test resourceVersion: "27970" uid: 686bdeb4-fe4c-4905-af7e-d6f16fa50b16 kind: Event lastTimestamp: "2026-07-28T17:44:56Z" message: 'Created pod: splitter-graph-824fb-7b8cd8fdfd-pdxxk' metadata: creationTimestamp: "2026-07-28T17:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:56Z" name: splitter-graph-824fb-7b8cd8fdfd.18c6852516f61f0e namespace: kserve-ci-e2e-test resourceVersion: "27978" uid: 2981a169-8aa6-4dd4-9fce-97e16adad119 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-824fb namespace: kserve-ci-e2e-test resourceVersion: "27968" uid: acc04b4f-0a7a-4376-a39b-1fd4da733146 kind: Event lastTimestamp: "2026-07-28T17:44:56Z" message: Scaled up replica set splitter-graph-824fb-7b8cd8fdfd from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:56Z" name: splitter-graph-824fb.18c685251584d99a namespace: kserve-ci-e2e-test resourceVersion: "27972" uid: 0c8dd64a-3803-4522-a5fd-892ddee56c26 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-824fb namespace: kserve-ci-e2e-test resourceVersion: "27988" uid: 8dc79273-1b75-4678-a64d-50f8e2290225 kind: Event lastTimestamp: "2026-07-28T17:44:56Z" message: InferenceGraph [splitter-graph-824fb] is Ready metadata: creationTimestamp: "2026-07-28T17:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:56Z" name: splitter-graph-824fb.18c685251b874fbb namespace: kserve-ci-e2e-test resourceVersion: "27990" uid: 4e1e5d63-b405-4cf9-a7bf-7acccfea6b8a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-824fb namespace: kserve-ci-e2e-test resourceVersion: "27969" uid: b287f439-fdae-4e97-8645-69de51fc24e4 kind: Event lastTimestamp: "2026-07-28T17:45:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:11Z" name: splitter-graph-824fb.18c6852893b77981 namespace: kserve-ci-e2e-test resourceVersion: "28116" uid: 21410e35-724a-459c-ade1-c917683df583 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-824fb namespace: kserve-ci-e2e-test resourceVersion: "27969" uid: b287f439-fdae-4e97-8645-69de51fc24e4 kind: Event lastTimestamp: "2026-07-28T17:45:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:11Z" name: splitter-graph-824fb.18c68528942239fc namespace: kserve-ci-e2e-test resourceVersion: "28119" uid: 70ad1571-1e13-4313-a22f-18d4c9747361 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-824fb namespace: kserve-ci-e2e-test resourceVersion: "28117" uid: b287f439-fdae-4e97-8645-69de51fc24e4 kind: Event lastTimestamp: "2026-07-28T17:45:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:26Z" name: splitter-graph-824fb.18c6852c1333a2b2 namespace: kserve-ci-e2e-test resourceVersion: "28228" uid: 1abd9235-4d64-4706-a961-bad96a374df0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:45:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-824fb namespace: kserve-ci-e2e-test resourceVersion: "28117" uid: b287f439-fdae-4e97-8645-69de51fc24e4 kind: Event lastTimestamp: "2026-07-28T17:45:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:26Z" name: splitter-graph-824fb.18c6852c139de4cd namespace: kserve-ci-e2e-test resourceVersion: "28230" uid: 84128a18-771b-4ce3-88d2-45611e0f9dba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:42.155610Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:42Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c68513bf14152d namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 6c9a2cbc-9c72-4b47-8d85-b63a4a4e1e79 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0dc16-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:42Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c68513c8652dc7 namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: c75df618-1155-4b52-bfd3-a26573df18f4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:43Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c68513fbf6ca35 namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 9ef9051c-6892-4ca5-8186-6fc3d9db7906 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c68513fd80d685 namespace: kserve-ci-e2e-test resourceVersion: "27082" uid: b11ae94e-1070-42f9-99fe-e1ffbf34275d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c6851400a87745 namespace: kserve-ci-e2e-test resourceVersion: "27083" uid: f23c83d4-7e05-4d87-b0d7-fd112a6ec32f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c6851401198b78 namespace: kserve-ci-e2e-test resourceVersion: "27084" uid: 0e509535-f799-4ef0-ae19-19b0701c5553 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c6851401237da7 namespace: kserve-ci-e2e-test resourceVersion: "27085" uid: ae962197-7c6c-4ced-b8d6-fe1e5f803b3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c6851403c953bc namespace: kserve-ci-e2e-test resourceVersion: "27086" uid: 9ec75e90-9a9d-4489-b28d-d3b082cc35b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T17:43:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:43Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c685140433893b namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 3f827dc4-1d52-40a8-9cda-0f36d9061a4e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T17:44:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:21Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c685146fa405db namespace: kserve-ci-e2e-test resourceVersion: "27614" uid: 903ebbda-9c98-44a3-84db-f06fb1c80003 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T18:01:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:01:01Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c68605ca460c7b namespace: kserve-ci-e2e-test resourceVersion: "34720" uid: e34dfaac-b76f-4d81-b951-9b0f2e44e31c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: f54c01fe-90bb-420a-9455-8577a038e7b4 kind: Event lastTimestamp: "2026-07-28T18:01:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:01:01Z" name: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5.18c68605ca46ad2c namespace: kserve-ci-e2e-test resourceVersion: "34721" uid: ffa7f788-16c7-491b-8ab5-4f1c8481be6e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0dc16-predictor-66967b5b55 namespace: kserve-ci-e2e-test resourceVersion: "27015" uid: 0250fde3-1e9c-4b28-b85d-0c9d472edbe1 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: 'Created pod: success-200-isvc-0dc16-predictor-66967b5b55-dpcm5' metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:42Z" name: success-200-isvc-0dc16-predictor-66967b5b55.18c68513be79f30e namespace: kserve-ci-e2e-test resourceVersion: "27027" uid: eded3886-627d-42ac-9df0-595b2ec39d91 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27012" uid: 68e08c50-37b0-4e3b-8e2f-f0f7f1cce532 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: Scaled up replica set success-200-isvc-0dc16-predictor-66967b5b55 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:42Z" name: success-200-isvc-0dc16-predictor.18c68513bc9827b1 namespace: kserve-ci-e2e-test resourceVersion: "27018" uid: f88617d8-d300-429e-a889-ef08cfbca401 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27016" uid: 24731c14-3910-49cc-9127-5044f538f56c kind: Event lastTimestamp: "2026-07-28T17:44:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:12Z" name: success-200-isvc-0dc16-predictor.18c685173ba1b4f6 namespace: kserve-ci-e2e-test resourceVersion: "27548" uid: db907236-37f5-484e-8382-1034ba9d42c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27016" uid: 24731c14-3910-49cc-9127-5044f538f56c kind: Event lastTimestamp: "2026-07-28T17:44:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:12Z" name: success-200-isvc-0dc16-predictor.18c685173c17669c namespace: kserve-ci-e2e-test resourceVersion: "27549" uid: 88ecd3b4-8b20-4bd4-92b3-0399eb73aa68 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:44:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27381" uid: 24731c14-3910-49cc-9127-5044f538f56c kind: Event lastTimestamp: "2026-07-28T17:44:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:57Z" name: success-200-isvc-0dc16-predictor.18c6851e3a11c28b namespace: kserve-ci-e2e-test resourceVersion: "28003" uid: 62152af6-88ce-4b2f-8b3d-3ad36ff0607e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:44:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dc16-predictor namespace: kserve-ci-e2e-test resourceVersion: "27381" uid: 24731c14-3910-49cc-9127-5044f538f56c kind: Event lastTimestamp: "2026-07-28T17:44:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:57Z" name: success-200-isvc-0dc16-predictor.18c6851e3a823a8a namespace: kserve-ci-e2e-test resourceVersion: "28004" uid: f10786ce-cf1c-4794-8eb0-31c60ed6dd45 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: a0b19f30-a008-4555-b222-519f09d77e41 kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:31Z" name: success-200-isvc-0dc16.18c68513bd1ebc04 namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: f69616a0-91c6-433c-a32a-c5e5dafc8565 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: a0b19f30-a008-4555-b222-519f09d77e41 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: 'Failed to update status for InferenceService "success-200-isvc-0dc16": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0dc16": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:42Z" name: success-200-isvc-0dc16.18c68513bd8a6005 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 43bc097e-9dfa-42b2-89c4-b9e97abaf018 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: a0b19f30-a008-4555-b222-519f09d77e41 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0dc16": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:42Z" name: success-200-isvc-0dc16.18c68513bd8ab389 namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: 4ac19fde-42f6-4f32-aa6a-0c264bf04651 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "27713" uid: a0b19f30-a008-4555-b222-519f09d77e41 kind: Event lastTimestamp: "2026-07-28T17:44:31Z" message: InferenceService [success-200-isvc-0dc16] is Ready metadata: creationTimestamp: "2026-07-28T17:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:31Z" name: success-200-isvc-0dc16.18c6851f2efb21c2 namespace: kserve-ci-e2e-test resourceVersion: "27714" uid: 0e5dbec4-3499-45f2-84dd-dcf49fd70446 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:42:46.251291Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26421" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-11994-predictor-5bbcdf967f-9qn7z to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68506baebe04a namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 9d8904c7-2c84-463e-8283-f9fc1492c14e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68506d446bf8f namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: 60c516b9-f5b1-484f-9bfe-41b9a82df4bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68506d5d7076b namespace: kserve-ci-e2e-test resourceVersion: "26482" uid: 7acb5431-c060-4349-8171-804227c6e872 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68506d8ab4105 namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: b604c059-bbad-4e66-95d5-6a77aaa8a0d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68506d9186d65 namespace: kserve-ci-e2e-test resourceVersion: "26486" uid: 2d9e5881-7a91-46ea-aeb0-d1386492a965 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68506d921f938 namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 584b6776-8ffb-4203-8079-fb28e4fed443 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68506dba9917f namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: 01002634-3dc5-475b-83b2-1d2dbea38472 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68506dc17c50d namespace: kserve-ci-e2e-test resourceVersion: "26493" uid: c3c93968-5958-4c1e-a471-9c62ca476ca5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:42:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:43:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:42:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:23Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c685071d5393d9 namespace: kserve-ci-e2e-test resourceVersion: "26804" uid: 15a1ea3b-a248-4638-a7c8-3a0dd3aa116d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:56Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c6851716900c6f namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: e0f43224-ff6b-4c63-b654-3e32a5b1a07e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:56Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68517169082bf namespace: kserve-ci-e2e-test resourceVersion: "27302" uid: fcaaee5a-02d6-4a5c-8592-6745d9a856a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: e8b0b08b-2f90-49b6-9c54-f2cfa08459fe kind: Event lastTimestamp: "2026-07-28T17:43:58Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:58Z" name: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z.18c68517a58f0ad3 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: a29753e9-6f92-4020-a49a-6e73526547e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-11994-predictor-5bbcdf967f namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 88d55094-8a64-45e6-8e9e-23e8f9c26683 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: 'Created pod: success-200-isvc-11994-predictor-5bbcdf967f-9qn7z' metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor-5bbcdf967f.18c68506ba78f118 namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 24dd194f-7c31-414c-8239-4f9799a60084 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26405" uid: d2462b65-82d1-41a3-a315-a1f9d57c36e8 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Scaled up replica set success-200-isvc-11994-predictor-5bbcdf967f from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994-predictor.18c68506b6ea8f9b namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: 888a6278-0049-4e7b-a3ca-3e7bd6690603 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: 3030dddb-6085-400a-960b-53a88879e820 kind: Event lastTimestamp: "2026-07-28T17:43:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:16Z" name: success-200-isvc-11994-predictor.18c6850a35cf24ab namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: c141c548-ddf8-461f-94e3-aa4711b3f210 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: 3030dddb-6085-400a-960b-53a88879e820 kind: Event lastTimestamp: "2026-07-28T17:43:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:16Z" name: success-200-isvc-11994-predictor.18c6850a36404c2d namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: fbc8d9c4-8d17-4f20-befe-e16b1049101c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 3030dddb-6085-400a-960b-53a88879e820 kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:46Z" name: success-200-isvc-11994-predictor.18c6851134325b85 namespace: kserve-ci-e2e-test resourceVersion: "27144" uid: 105fbdb2-be7c-41a2-9e79-cb2e0b87eb90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11994-predictor namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 3030dddb-6085-400a-960b-53a88879e820 kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:46Z" name: success-200-isvc-11994-predictor.18c6851134a52e1c namespace: kserve-ci-e2e-test resourceVersion: "27146" uid: 87a6623c-0c6e-41d3-bb54-c1c372af9696 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11994 namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: ba2fdf80-7827-403d-8034-7542b954680d kind: Event lastTimestamp: "2026-07-28T17:43:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:34Z" name: success-200-isvc-11994.18c68506b730ae7d namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: d9daf6b0-c6bc-44bb-81cf-05f284b928cc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11994 namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: ba2fdf80-7827-403d-8034-7542b954680d kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: 'Failed to update status for InferenceService "success-200-isvc-11994": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-11994": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994.18c68506b7a01188 namespace: kserve-ci-e2e-test resourceVersion: "26416" uid: 86d0cec4-7748-4c21-bef6-7834bd08f2ab reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11994 namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: ba2fdf80-7827-403d-8034-7542b954680d kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-11994": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-11994.18c68506b7a06e83 namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: 13a472f1-ef44-4b4c-b5cc-cfe03ec73e02 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11994 namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: ba2fdf80-7827-403d-8034-7542b954680d kind: Event lastTimestamp: "2026-07-28T17:43:34Z" message: InferenceService [success-200-isvc-11994] is Ready metadata: creationTimestamp: "2026-07-28T17:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:43:34Z" name: success-200-isvc-11994.18c68511d91bb7a3 namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: 142cc38d-21e8-4cec-9b8b-c151dd3cf8b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:24:01.980804Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17115" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-211cd-predictor-5f9579f849-vpgx8 to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:24:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:24:01Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c68400f72dba53 namespace: kserve-ci-e2e-test resourceVersion: "17122" uid: 4d3333e7-40b1-446f-864f-fbbf1d0a27fc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-211cd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c68401025e268e namespace: kserve-ci-e2e-test resourceVersion: "17196" uid: 93eba1ff-33c5-4ed1-bb1c-0a7ff42aced7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17127" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:24:03Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c684013482717c namespace: kserve-ci-e2e-test resourceVersion: "17233" uid: ad8b7c02-98fe-4d90-89e5-6a9600507bdf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:03Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c6840135f6f1ec namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: 29eec550-792e-4c0a-a4cd-55fa667708fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:03Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c684013998cfca namespace: kserve-ci-e2e-test resourceVersion: "17237" uid: b4c5ca66-ab29-4265-b352-09716e59919b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:03Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c684013a45d076 namespace: kserve-ci-e2e-test resourceVersion: "17238" uid: 9eb0ddf9-3d7e-4b9d-a414-adcc4c1ddc5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:03Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c684013a51cc27 namespace: kserve-ci-e2e-test resourceVersion: "17239" uid: 8c3fe729-9b8a-430a-b12a-cd497a87deed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:03Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c684013e9ca690 namespace: kserve-ci-e2e-test resourceVersion: "17243" uid: 742b272d-7289-4148-a11d-b4300cad3578 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:03Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c684013f2846b8 namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: 7d11acfd-dd51-49a8-bea8-b81f976bfcf2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:33:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:19Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c684016340dea0 namespace: kserve-ci-e2e-test resourceVersion: "21567" uid: 3db8cc3d-7bd5-46ce-a2dd-9badda86818c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:33:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:16Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c6848224ee2eb3 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 6facdc11-5960-4db7-8322-89b16362efa2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:33:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:16Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c6848224eebd86 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 421f041b-c74e-4751-80db-47aecb2972c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 37d036fa-be96-4ebe-a842-4df4ebb4355a kind: Event lastTimestamp: "2026-07-28T17:33:19Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:19Z" name: success-200-isvc-211cd-predictor-5f9579f849-vpgx8.18c68482d7a33e7e namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: ab293e61-5c53-4540-87e7-086b4de09d0d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-211cd-predictor-5f9579f849 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 35a48812-3acc-462d-9904-564471e4f56e kind: Event lastTimestamp: "2026-07-28T17:24:01Z" message: 'Created pod: success-200-isvc-211cd-predictor-5f9579f849-vpgx8' metadata: creationTimestamp: "2026-07-28T17:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:01Z" name: success-200-isvc-211cd-predictor-5f9579f849.18c68400f68f4178 namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: 27f50696-bf50-4ebb-9159-70dc97e1b4f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17090" uid: 5056b10d-d6cc-43db-873d-a2aef0a17fd8 kind: Event lastTimestamp: "2026-07-28T17:24:01Z" message: Scaled up replica set success-200-isvc-211cd-predictor-5f9579f849 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:01Z" name: success-200-isvc-211cd-predictor.18c68400f4aa187d namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: d0fd0faa-764b-4c96-a373-5cb2ae65d34f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 4943bad3-580e-41da-9a31-5eb29d3e9cc7 kind: Event lastTimestamp: "2026-07-28T17:24:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:16Z" name: success-200-isvc-211cd-predictor.18c6840473b21378 namespace: kserve-ci-e2e-test resourceVersion: "17384" uid: 46f166f8-51a9-4e02-9f1e-7427b806ace5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 4943bad3-580e-41da-9a31-5eb29d3e9cc7 kind: Event lastTimestamp: "2026-07-28T17:24:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:16Z" name: success-200-isvc-211cd-predictor.18c6840474217195 namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: a72a3cc2-e0c4-463b-ba11-1522b1a876c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:24:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17385" uid: 4943bad3-580e-41da-9a31-5eb29d3e9cc7 kind: Event lastTimestamp: "2026-07-28T17:25:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:25:17Z" name: success-200-isvc-211cd-predictor.18c68407f2d18560 namespace: kserve-ci-e2e-test resourceVersion: "17898" uid: ae5bce87-c2e1-4a6f-9b22-1ae2a8d4a777 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:24:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-211cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "17385" uid: 4943bad3-580e-41da-9a31-5eb29d3e9cc7 kind: Event lastTimestamp: "2026-07-28T17:25:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:25:17Z" name: success-200-isvc-211cd-predictor.18c68407f33b61b9 namespace: kserve-ci-e2e-test resourceVersion: "17899" uid: 3de12e63-dc24-4313-b5eb-487ff1090695 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:24:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-211cd namespace: kserve-ci-e2e-test resourceVersion: "17072" uid: 79c0bf29-06d4-4c9d-aea6-11bf24afda7e kind: Event lastTimestamp: "2026-07-28T17:24:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:24:49Z" name: success-200-isvc-211cd.18c68400f4bffd82 namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: e85e7571-e2a1-4a95-8a0c-874a2c41651a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-211cd namespace: kserve-ci-e2e-test resourceVersion: "17662" uid: 79c0bf29-06d4-4c9d-aea6-11bf24afda7e kind: Event lastTimestamp: "2026-07-28T17:24:49Z" message: InferenceService [success-200-isvc-211cd] is Ready metadata: creationTimestamp: "2026-07-28T17:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:24:49Z" name: success-200-isvc-211cd.18c6840c1fa5c2be namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 1140520d-45b1-4853-a23b-ceb37bdb2d09 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:23:02.069180Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-391b7-predictor-888b7969f-kw99r to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:23:02Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f3042ae962 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: da039d44-23d2-4cd7-8d0c-99794f355453 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-391b7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f3121c1a30 namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 6fd842ab-1275-467c-a850-1ce4ad275acb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:23:03Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:23:03Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f3411a56ca namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: ead96181-a31a-4b4b-9858-b9467b39ac50 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:23:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:03Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f342884383 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: 35d179a1-a852-47a1-8f17-5d6a70554281 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:23:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:03Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f3455a1ab0 namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: bf1748d8-1328-4f8f-a756-73ec91584aaf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:23:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:03Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f345bc4738 namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: a823e45c-e1d2-467a-b953-cef906153061 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:23:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:03Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f345c60283 namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: a87bf3b6-aade-400c-a2d9-21d934167e97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:23:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:03Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f34881d0fd namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 1733c208-4876-49e1-99ff-fcaf90eb7663 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:23:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:03Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f348e0427c namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 4bb3bb2c-3bd9-48ac-b640-5a3075a3c8f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:23:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:40Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c683f3996feef7 namespace: kserve-ci-e2e-test resourceVersion: "16835" uid: 833d703c-48b7-4b6e-b413-66e272ecff71 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:32:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:16Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c68474374b2c16 namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 4d4cb6d3-3f5e-423d-b43d-25fdc5169b97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-391b7-predictor-888b7969f-kw99r namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 01b01629-5732-466b-809b-4fd5067df411 kind: Event lastTimestamp: "2026-07-28T17:32:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:16Z" name: success-200-isvc-391b7-predictor-888b7969f-kw99r.18c68474374bc3c1 namespace: kserve-ci-e2e-test resourceVersion: "20807" uid: b8f5eccc-0888-497d-b5c8-9c30d9928cb0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-391b7-predictor-888b7969f namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: d8a0fab5-ce4c-4645-8ced-43e6132059b3 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: 'Created pod: success-200-isvc-391b7-predictor-888b7969f-kw99r' metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:02Z" name: success-200-isvc-391b7-predictor-888b7969f.18c683f303965baf namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: d6da0aa9-77a0-4580-a138-89efa061b6bd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: 57d5cc15-61a1-4eee-93e0-489f9cabdac0 kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Scaled up replica set success-200-isvc-391b7-predictor-888b7969f from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:02Z" name: success-200-isvc-391b7-predictor.18c683f30190618c namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 17f861a2-f134-4f60-a5a6-d8b1a438df4d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 8c391417-b347-43a7-8c24-4a1e8a43b6d8 kind: Event lastTimestamp: "2026-07-28T17:23:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:17Z" name: success-200-isvc-391b7-predictor.18c683f6809d9715 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 81581636-ddc4-410e-82e4-5688771f955e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 8c391417-b347-43a7-8c24-4a1e8a43b6d8 kind: Event lastTimestamp: "2026-07-28T17:23:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:17Z" name: success-200-isvc-391b7-predictor.18c683f681263468 namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 06351811-4437-4d8c-8aa8-7b773148c424 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:23:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 8c391417-b347-43a7-8c24-4a1e8a43b6d8 kind: Event lastTimestamp: "2026-07-28T17:24:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:17Z" name: success-200-isvc-391b7-predictor.18c683f9ffccf774 namespace: kserve-ci-e2e-test resourceVersion: "17391" uid: ed60d4a8-81c3-4484-990f-dc7d1d503b03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:23:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-391b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 8c391417-b347-43a7-8c24-4a1e8a43b6d8 kind: Event lastTimestamp: "2026-07-28T17:24:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:23:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:17Z" name: success-200-isvc-391b7-predictor.18c683fa003aea00 namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: fe8e0dbf-bb6d-43e3-92fc-a6c9091ef104 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-391b7 namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 6bff9e0c-f695-4a1c-8a95-3935e28a1112 kind: Event lastTimestamp: "2026-07-28T17:23:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:50Z" name: success-200-isvc-391b7.18c683f30239cef0 namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: 1f73cf1d-ec96-49ae-b29d-f79f27f206a1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-391b7 namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: 6bff9e0c-f695-4a1c-8a95-3935e28a1112 kind: Event lastTimestamp: "2026-07-28T17:23:50Z" message: InferenceService [success-200-isvc-391b7] is Ready metadata: creationTimestamp: "2026-07-28T17:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:23:50Z" name: success-200-isvc-391b7.18c683fe553174a5 namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: 9f541037-4175-4504-aba1-5b9ee51e152c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:32:17.044039Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:32:17Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c684743b3f1f2e namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: fd8fd150-f942-43c6-850f-797af402b9b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:32:17Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c6847454c673fc namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: fab6cf62-424a-487d-adce-60707e8773f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c68474564556b0 namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: a60b848e-94b3-4121-b458-d3a6e22bdcb3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c6847458f87b91 namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: a61f9a2c-28c6-4896-8e66-3bbaf058e7db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c68474596e337e namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 4a1ee2e0-0303-4f15-8fba-5986a36ab938 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c6847459788281 namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 5e2db05c-751d-4a44-ba4b-f5f193b2a194 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c684745f774208 namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: a2cbaf92-bb1a-436d-9683-4a488af3d7c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:17Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c684745fec46e7 namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: aa97e745-c2dd-491b-bfe8-084a36034064 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:33:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:34Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c6847499befebd namespace: kserve-ci-e2e-test resourceVersion: "21844" uid: 6a1b5662-4fea-4bd7-bf52-1547bbce3568 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:31Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c68485883bfd5f namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: a8199fd5-814f-4f18-9110-266120acccab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:31Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c68485883ca364 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 01eac8d4-3e67-470e-8933-e1879c977ef4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: e4da259a-bf2f-41d2-98c4-1ab1272c65de kind: Event lastTimestamp: "2026-07-28T17:33:34Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:34Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz.18c684864be08952 namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: 3bf8a5f4-ab52-4019-9861-def649d9b160 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4a23a-predictor-64fbf5fcd5 namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 8c9674a4-158e-49aa-9da8-568bda620c13 kind: Event lastTimestamp: "2026-07-28T17:32:17Z" message: 'Created pod: success-200-isvc-4a23a-predictor-64fbf5fcd5-d5bxz' metadata: creationTimestamp: "2026-07-28T17:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:17Z" name: success-200-isvc-4a23a-predictor-64fbf5fcd5.18c684743abc2ddd namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: 08ec0f85-8379-4704-97f6-f1fdd3078174 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 6eb13e89-1b36-4429-9bf3-d10d928882f1 kind: Event lastTimestamp: "2026-07-28T17:32:16Z" message: Scaled up replica set success-200-isvc-4a23a-predictor-64fbf5fcd5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:16Z" name: success-200-isvc-4a23a-predictor.18c68474371b71ce namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: 2db58e5e-1beb-4168-be65-def6dc8221a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: 5e600b5f-5280-4854-ae97-baac98b1984c kind: Event lastTimestamp: "2026-07-28T17:32:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:31Z" name: success-200-isvc-4a23a-predictor.18c68477b58d285a namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: 3c577e7e-69b9-4eda-8cfd-c0d49ea6a4f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: 5e600b5f-5280-4854-ae97-baac98b1984c kind: Event lastTimestamp: "2026-07-28T17:32:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:32:31Z" name: success-200-isvc-4a23a-predictor.18c68477b5f8fc7f namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 6306bf9b-6dea-45e8-8ad2-5891af87f839 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:32:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 5e600b5f-5280-4854-ae97-baac98b1984c kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:17Z" name: success-200-isvc-4a23a-predictor.18c6847b35199268 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: 68b73f0c-9bd3-40e8-a15a-ea950d4fdb09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:32:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a23a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 5e600b5f-5280-4854-ae97-baac98b1984c kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:17Z" name: success-200-isvc-4a23a-predictor.18c6847b35a8a14e namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: dcab54f1-4d6f-4efb-ba4d-1eb2fe4f78e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:32:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a23a namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: f33893ab-b072-49c1-ac87-63d9c39347ea kind: Event lastTimestamp: "2026-07-28T17:33:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:04Z" name: success-200-isvc-4a23a.18c6847436ed3dbf namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: c2144320-9cfb-4924-b9ff-1bb2732ea624 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a23a namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: f33893ab-b072-49c1-ac87-63d9c39347ea kind: Event lastTimestamp: "2026-07-28T17:32:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-4a23a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4a23a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:16Z" name: success-200-isvc-4a23a.18c684743757811c namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: e127f2c3-43e4-4720-a7ee-18c64b8bef10 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a23a namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: f33893ab-b072-49c1-ac87-63d9c39347ea kind: Event lastTimestamp: "2026-07-28T17:32:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4a23a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:32:16Z" name: success-200-isvc-4a23a.18c684743757dfba namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 23734363-17cc-4b42-999f-c5fe7e7f9bc3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a23a namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: f33893ab-b072-49c1-ac87-63d9c39347ea kind: Event lastTimestamp: "2026-07-28T17:33:04Z" message: InferenceService [success-200-isvc-4a23a] is Ready metadata: creationTimestamp: "2026-07-28T17:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:04Z" name: success-200-isvc-4a23a.18c6847f54f270af namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 6e2be81e-909a-4791-a9df-fb33e4428a0e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:43:56.559098Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:43:56Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c685171997fbc4 namespace: kserve-ci-e2e-test resourceVersion: "27327" uid: 49ad276c-1756-4d31-8f61-75b61b76f946 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-824fb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:56Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c685172982ee86 namespace: kserve-ci-e2e-test resourceVersion: "27371" uid: fa2e9538-d611-49b0-bacf-b95dd35be2a9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27336" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:43:57Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c6851756b3ab09 namespace: kserve-ci-e2e-test resourceVersion: "27388" uid: 385c959e-2148-4d63-87e2-6cacfd7f2a7e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c685175825ec02 namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: bf2e8492-14e2-49f9-a3e9-89074296bf1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c685175b3f819c namespace: kserve-ci-e2e-test resourceVersion: "27392" uid: 9931b15b-6581-414b-915d-c26f24da55ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c685175bc1eec1 namespace: kserve-ci-e2e-test resourceVersion: "27393" uid: 58861909-84bd-4001-b680-513457f0a51a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c685175bcc00ba namespace: kserve-ci-e2e-test resourceVersion: "27394" uid: 4f5f3cf4-5a9f-4a55-927c-8086b6d4203e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c685175e833736 namespace: kserve-ci-e2e-test resourceVersion: "27395" uid: c90fd554-ee1a-4f30-a7fc-5a819198c98f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:43:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:57Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c685175eef5e1f namespace: kserve-ci-e2e-test resourceVersion: "27397" uid: 687b816b-063f-4b57-8337-56f7edab8aaf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:53:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:14Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c6851779823cf9 namespace: kserve-ci-e2e-test resourceVersion: "31508" uid: 2915baec-2cd4-4171-8b36-02e13e8ef481 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:53:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:11Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c6859843ca7548 namespace: kserve-ci-e2e-test resourceVersion: "31466" uid: 835323a1-fe46-41be-a065-a6460389ba9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:53:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:11Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c6859843cbc723 namespace: kserve-ci-e2e-test resourceVersion: "31467" uid: 1918d8f0-d902-4780-9843-f5296972d95c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:53:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 66d41b88-450f-4ffe-a9a5-8532f9e04040 kind: Event lastTimestamp: "2026-07-28T17:53:14Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:53:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:53:14Z" name: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr.18c68598edee6903 namespace: kserve-ci-e2e-test resourceVersion: "31507" uid: 8cc341ab-0ca5-489e-8c8e-299b83249dd7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-824fb-predictor-6f7f8d49f namespace: kserve-ci-e2e-test resourceVersion: "27303" uid: fc437dad-7ce1-4dbb-a711-5910030d787c kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: 'Created pod: success-200-isvc-824fb-predictor-6f7f8d49f-zmdrr' metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:56Z" name: success-200-isvc-824fb-predictor-6f7f8d49f.18c68517194ab53d namespace: kserve-ci-e2e-test resourceVersion: "27322" uid: f759f20a-3614-4676-93b2-16302eefda48 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27297" uid: a18cee86-6abe-4f6d-9bf0-bed7a6071a29 kind: Event lastTimestamp: "2026-07-28T17:43:56Z" message: Scaled up replica set success-200-isvc-824fb-predictor-6f7f8d49f from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:43:56Z" name: success-200-isvc-824fb-predictor.18c6851717521852 namespace: kserve-ci-e2e-test resourceVersion: "27305" uid: 04374724-f6df-41a1-b215-7612412fc739 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27301" uid: 1f905296-b0f9-4e46-a171-f45e4bc98299 kind: Event lastTimestamp: "2026-07-28T17:44:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:26Z" name: success-200-isvc-824fb-predictor.18c6851a96229886 namespace: kserve-ci-e2e-test resourceVersion: "27652" uid: 86184414-9094-4d7a-acff-f974375ada79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27301" uid: 1f905296-b0f9-4e46-a171-f45e4bc98299 kind: Event lastTimestamp: "2026-07-28T17:44:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:26Z" name: success-200-isvc-824fb-predictor.18c6851a9693b37e namespace: kserve-ci-e2e-test resourceVersion: "27653" uid: 84ac6550-c74d-4efa-93c8-5ae318b72b70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27534" uid: 1f905296-b0f9-4e46-a171-f45e4bc98299 kind: Event lastTimestamp: "2026-07-28T17:44:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:56Z" name: success-200-isvc-824fb-predictor.18c6852194ecdf9c namespace: kserve-ci-e2e-test resourceVersion: "27971" uid: 698ad9f3-e6ef-4372-93e6-572b9400ea11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-824fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27534" uid: 1f905296-b0f9-4e46-a171-f45e4bc98299 kind: Event lastTimestamp: "2026-07-28T17:44:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:56Z" name: success-200-isvc-824fb-predictor.18c685219557413e namespace: kserve-ci-e2e-test resourceVersion: "27975" uid: 48e0aee5-dc47-4b82-a5d3-4d94d861be58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:43:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-824fb namespace: kserve-ci-e2e-test resourceVersion: "27279" uid: 9ec7927f-14fa-4996-aba7-6ec616d50329 kind: Event lastTimestamp: "2026-07-28T17:44:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:44Z" name: success-200-isvc-824fb.18c685171758223a namespace: kserve-ci-e2e-test resourceVersion: "27863" uid: 4c43946b-23b7-4c16-afcb-d593b59bf5e5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-824fb namespace: kserve-ci-e2e-test resourceVersion: "27860" uid: 9ec7927f-14fa-4996-aba7-6ec616d50329 kind: Event lastTimestamp: "2026-07-28T17:44:44Z" message: InferenceService [success-200-isvc-824fb] is Ready metadata: creationTimestamp: "2026-07-28T17:44:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:44Z" name: success-200-isvc-824fb.18c685223582ae52 namespace: kserve-ci-e2e-test resourceVersion: "27861" uid: 9c34b738-028c-498d-90e0-781a48526ed0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:34:27.257899Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22413" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a513a-predictor-5477b77665-h87n4 to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c684928c990949 namespace: kserve-ci-e2e-test resourceVersion: "22420" uid: 0514b0d1-4151-4934-8219-4c9980875a51 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22426" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68492a68da290 namespace: kserve-ci-e2e-test resourceVersion: "22466" uid: aa772311-40e1-47ac-a534-bb5d1be6e084 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68492a7f1d077 namespace: kserve-ci-e2e-test resourceVersion: "22468" uid: ead55a0f-6f51-4b9d-b103-f393b186a617 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68492aac06696 namespace: kserve-ci-e2e-test resourceVersion: "22470" uid: e78b8458-c61e-4fb5-8a1b-5935ea94eeb1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68492ab3321c8 namespace: kserve-ci-e2e-test resourceVersion: "22471" uid: b1a540ab-b366-4059-8822-ada45aeca54c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68492ab3d6f98 namespace: kserve-ci-e2e-test resourceVersion: "22472" uid: e709f10d-2458-4ece-aec6-abbcd7ea0635 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68492aff22a76 namespace: kserve-ci-e2e-test resourceVersion: "22473" uid: 7998b67c-e75a-467a-b64e-ba6757dbe29d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68492b066c087 namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: 4a29f44d-0029-4507-bf12-f42d3728ad76 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:35:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:35:05Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68492fd5a92bc namespace: kserve-ci-e2e-test resourceVersion: "22843" uid: 7a7038a9-25f8-428d-86ed-05e8cd3b534e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:42Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68513bb61812e namespace: kserve-ci-e2e-test resourceVersion: "27013" uid: 5727c0b5-b4a7-4447-bf17-d13a4054b977 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:43:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:42Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68513bb622e20 namespace: kserve-ci-e2e-test resourceVersion: "27014" uid: 5e8b392c-e63a-43a2-8eb1-92faad091af1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:43:45Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:45Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c6851471b281e4 namespace: kserve-ci-e2e-test resourceVersion: "27121" uid: 9fa7b51c-00fe-4613-8faa-282811599a3f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a513a-predictor-5477b77665-h87n4 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 5dadb749-2e06-4e5c-9cc3-ac42790048f3 kind: Event lastTimestamp: "2026-07-28T17:43:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: i/o timeout' metadata: creationTimestamp: "2026-07-28T17:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:43:46Z" name: success-200-isvc-a513a-predictor-5477b77665-h87n4.18c68514ad90df42 namespace: kserve-ci-e2e-test resourceVersion: "27137" uid: 78a02312-6b0e-4fd7-89c0-fc16ce9a953a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a513a-predictor-5477b77665 namespace: kserve-ci-e2e-test resourceVersion: "22399" uid: d0919b3a-7d3f-4dc0-8e2e-1ccd38839605 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: 'Created pod: success-200-isvc-a513a-predictor-5477b77665-h87n4' metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor-5477b77665.18c684928c283bd3 namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: e42cb9d6-c620-485c-9efb-32d0aa938db2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22392" uid: 1e718575-e1fe-46c0-9a90-05abad11c841 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Scaled up replica set success-200-isvc-a513a-predictor-5477b77665 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a-predictor.18c684928a5be372 namespace: kserve-ci-e2e-test resourceVersion: "22401" uid: edbf9874-cf2b-40b7-8685-a09689441f6a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:34:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22394" uid: c6496eaa-46cf-411e-a3f3-ab1781213bab kind: Event lastTimestamp: "2026-07-28T17:34:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:57Z" name: success-200-isvc-a513a-predictor.18c6849608df6dce namespace: kserve-ci-e2e-test resourceVersion: "22777" uid: 69ec29e4-9620-45e8-aada-8b12ecf033b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:34:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22394" uid: c6496eaa-46cf-411e-a3f3-ab1781213bab kind: Event lastTimestamp: "2026-07-28T17:34:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:57Z" name: success-200-isvc-a513a-predictor.18c684960956a28d namespace: kserve-ci-e2e-test resourceVersion: "22778" uid: b51671ee-8c88-48c3-a6fe-1431de3dbdfc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:35:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22657" uid: c6496eaa-46cf-411e-a3f3-ab1781213bab kind: Event lastTimestamp: "2026-07-28T17:35:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:42Z" name: success-200-isvc-a513a-predictor.18c6849d070a42d3 namespace: kserve-ci-e2e-test resourceVersion: "23166" uid: 1c79e76f-d913-4cb8-9c40-d302eb5f0c56 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:35:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a513a-predictor namespace: kserve-ci-e2e-test resourceVersion: "22657" uid: c6496eaa-46cf-411e-a3f3-ab1781213bab kind: Event lastTimestamp: "2026-07-28T17:35:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:35:42Z" name: success-200-isvc-a513a-predictor.18c6849d07add121 namespace: kserve-ci-e2e-test resourceVersion: "23167" uid: 03143ac0-aeb1-47e3-804d-b11961e29237 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a513a namespace: kserve-ci-e2e-test resourceVersion: "22371" uid: e3b9a35b-b43d-479e-ae3a-acafd6c406bc kind: Event lastTimestamp: "2026-07-28T17:35:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:15Z" name: success-200-isvc-a513a.18c684928a867f1d namespace: kserve-ci-e2e-test resourceVersion: "22924" uid: af8c9cc2-89d9-4b78-a838-2e89014a792e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a513a namespace: kserve-ci-e2e-test resourceVersion: "22371" uid: e3b9a35b-b43d-479e-ae3a-acafd6c406bc kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: 'Failed to update status for InferenceService "success-200-isvc-a513a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a513a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a.18c684928b59a2a0 namespace: kserve-ci-e2e-test resourceVersion: "22411" uid: a447b76e-ced2-493b-94ec-4d071f0771e5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a513a namespace: kserve-ci-e2e-test resourceVersion: "22371" uid: e3b9a35b-b43d-479e-ae3a-acafd6c406bc kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a513a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-a513a.18c684928b5a1467 namespace: kserve-ci-e2e-test resourceVersion: "22414" uid: b76a1bb2-8096-4f47-8afe-5802f32b549c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:35:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a513a namespace: kserve-ci-e2e-test resourceVersion: "22921" uid: e3b9a35b-b43d-479e-ae3a-acafd6c406bc kind: Event lastTimestamp: "2026-07-28T17:35:15Z" message: InferenceService [success-200-isvc-a513a] is Ready metadata: creationTimestamp: "2026-07-28T17:35:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:35:15Z" name: success-200-isvc-a513a.18c6849db92f52e2 namespace: kserve-ci-e2e-test resourceVersion: "22923" uid: 7aaaf723-48db-4f30-9a30-cf91b8f5e828 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:21:31.672883Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:21:31Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683ddf8215efd namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 6aaecdad-05c9-447f-a36e-fd70e8deb62c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b94d0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:31Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683de05679473 namespace: kserve-ci-e2e-test resourceVersion: "15366" uid: e3c98e09-d484-4f0b-8c1c-ac6a0f8e6000 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:21:32Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683de35a1657d namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 9d3407d6-9789-440d-9650-ad96afc392b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:32Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T17:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:32Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683de37195b9c namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: abbb62c6-0f05-4dc6-bd76-7ab63c13df7a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:44Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" in 11.432s (11.432s including waiting). Image size: 1334926941 bytes.' metadata: creationTimestamp: "2026-07-28T17:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:44Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683e0e08ea29c namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 64db8ddf-a530-4d9a-9635-b63f003d66c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:44Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683e0e402363e namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: a1d29194-4fb6-487d-a3c2-055c30030e57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:44Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683e0e4a6dc88 namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: ab56495f-0837-4ca6-a589-cd42f1d290ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:44Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683e0e4b1b116 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 46e50f67-959e-4272-9ccf-5b23f19f919c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:46Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.258s (2.258s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:46Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683e16b4a766a namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: 90eb33a8-adc4-4a7c-830e-b0dc7e882245 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:46Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683e16ebe82eb namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: ad50a5fe-6a4d-4fa0-9c46-ad251afe879a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:21:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:21:46Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683e16f39a465 namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: fc9ab6fa-7179-45fd-bd76-6cd37c073fda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:21:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:23:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:03Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683e19ab5ebe8 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: ee5936c1-3eec-4a3f-8bc9-a1893409b824 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683f3004d1595 namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: bfcc84e1-e5ce-48a7-8948-2d044c563bc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:23:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:02Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683f3004daa05 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 07935edd-75bc-406e-bbc4-11a65d45ad9a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 5ff3495b-f4af-49c3-be73-58aa5137a2bb kind: Event lastTimestamp: "2026-07-28T17:23:03Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:03Z" name: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7.18c683f34cd3c9e6 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 5308f95f-2619-4a6d-9318-83d308eb5ff8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b94d0-predictor-d7574d6d9 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: cca57802-64a7-4a51-9c6c-bfa9891ddcdd kind: Event lastTimestamp: "2026-07-28T17:21:31Z" message: 'Created pod: success-200-isvc-b94d0-predictor-d7574d6d9-nd2n7' metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:31Z" name: success-200-isvc-b94d0-predictor-d7574d6d9.18c683ddf7979f55 namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: f7ddea8f-98e4-4bd0-a2bd-95df6081ad1b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: 87ffd20a-e437-40a5-920b-f86ba0abad07 kind: Event lastTimestamp: "2026-07-28T17:21:31Z" message: Scaled up replica set success-200-isvc-b94d0-predictor-d7574d6d9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:21:31Z" name: success-200-isvc-b94d0-predictor.18c683ddf0687627 namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 7a0beac0-c338-4193-8ebe-c81fb16ce143 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 86df02b1-923b-44e8-bf0f-91f21a29e882 kind: Event lastTimestamp: "2026-07-28T17:22:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:16Z" name: success-200-isvc-b94d0-predictor.18c683e175f4afe0 namespace: kserve-ci-e2e-test resourceVersion: "15920" uid: ec36a4a1-881f-408b-91e1-3d6c33ef4ed0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:21:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 86df02b1-923b-44e8-bf0f-91f21a29e882 kind: Event lastTimestamp: "2026-07-28T17:22:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:16Z" name: success-200-isvc-b94d0-predictor.18c683e1766a0a91 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: bcc74faa-fc61-416a-bf2f-ba50fcb67da0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:22:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 86df02b1-923b-44e8-bf0f-91f21a29e882 kind: Event lastTimestamp: "2026-07-28T17:23:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:01Z" name: success-200-isvc-b94d0-predictor.18c683ebf34395e3 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 377f6c87-07ff-4ea4-88b7-cf8222e28abd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:22:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b94d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 86df02b1-923b-44e8-bf0f-91f21a29e882 kind: Event lastTimestamp: "2026-07-28T17:23:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:23:01Z" name: success-200-isvc-b94d0-predictor.18c683ebf3afc326 namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: 3e86a6bf-f8a2-4523-94b8-593936f2dfb0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b94d0 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 8bd286b8-a8dd-4812-899a-51d50c4ee551 kind: Event lastTimestamp: "2026-07-28T17:22:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:33Z" name: success-200-isvc-b94d0.18c683ddf67d45b3 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 0e6de383-80b4-4397-80dc-ac1486ce63fd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b94d0 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 8bd286b8-a8dd-4812-899a-51d50c4ee551 kind: Event lastTimestamp: "2026-07-28T17:21:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-b94d0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b94d0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:21:31Z" name: success-200-isvc-b94d0.18c683ddf6eb585e namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: dc1ae86b-8fc9-4701-b675-a2de936c1f5d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:21:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b94d0 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 8bd286b8-a8dd-4812-899a-51d50c4ee551 kind: Event lastTimestamp: "2026-07-28T17:21:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b94d0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:21:31Z" name: success-200-isvc-b94d0.18c683ddf6eba2ab namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: d2bb0f9e-4092-4b8c-b79d-5477f5eb133d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b94d0 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 8bd286b8-a8dd-4812-899a-51d50c4ee551 kind: Event lastTimestamp: "2026-07-28T17:22:33Z" message: InferenceService [success-200-isvc-b94d0] is Ready metadata: creationTimestamp: "2026-07-28T17:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:33Z" name: success-200-isvc-b94d0.18c683ec5758a6dd namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: ceae79d0-1547-4334-a055-63650d35de91 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:31.397949Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:31Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c684858b15cdb7 namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 64a09376-9464-4711-a5ac-44c6d18734c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d1fc0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:31Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68485968481a9 namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: da6e3239-0945-4f97-bce5-1b19b9d9b708 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:33:32Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68485c87c3303 namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: 9370b2a6-5270-4c8a-8ae1-65cab9508818 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68485ca0c05eb namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: 50369c4b-48ed-4b3e-8b2c-632dada0e259 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68485cce0e81b namespace: kserve-ci-e2e-test resourceVersion: "21814" uid: 1473da98-d186-496b-9d11-0e5b7fa016a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68485cd4c61dd namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: 5cea9b28-4da4-43fb-b71d-a4364ac54108 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68485cd564f64 namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: 05659e38-84f3-43b0-b1da-f0cc9a84e1f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68485d0ebcaab namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: 7b1afa03-f874-421f-adb9-2e3e257bd637 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:33:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:32Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68485d1610797 namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 51053c97-bab7-4e1f-8a84-84e104ae4de4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:34:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:09Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c684862218bfb6 namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: b9c894d6-af32-43d9-b5c1-9eda7722e97d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68506b6f54a69 namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: e7655620-1b65-4677-97d0-c85dffcf1f89 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 8516fcf2-f6ee-4544-9dd7-56ccda7afae2 kind: Event lastTimestamp: "2026-07-28T17:42:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:42:46Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd.18c68506b6f85dfb namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 88a0075c-1d6d-4b8e-9d70-8f76e45edca5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d1fc0-predictor-6d9fcbc679 namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: 5fee6460-94a3-483f-a45f-191d4692f923 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'Created pod: success-200-isvc-d1fc0-predictor-6d9fcbc679-5w7nd' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:31Z" name: success-200-isvc-d1fc0-predictor-6d9fcbc679.18c684858a7dae43 namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 3ba8e2e5-6124-44cc-b5d2-59c0fb359ce8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 70de8e26-6847-4f17-be6a-03f111a4ab15 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: Scaled up replica set success-200-isvc-d1fc0-predictor-6d9fcbc679 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:31Z" name: success-200-isvc-d1fc0-predictor.18c6848588de8162 namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: 4fdc51cb-52bf-41d7-8b1d-0bbb909a194d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 504e80b1-8a90-4eed-9f55-d8faf573442b kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:46Z" name: success-200-isvc-d1fc0-predictor.18c6848907db6bd5 namespace: kserve-ci-e2e-test resourceVersion: "21953" uid: ff9e28e4-8dab-4fe7-b2dc-815874d0b32d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 504e80b1-8a90-4eed-9f55-d8faf573442b kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:46Z" name: success-200-isvc-d1fc0-predictor.18c68489084630ac namespace: kserve-ci-e2e-test resourceVersion: "21955" uid: 6e1ed2bb-0560-4b61-86f4-bbadb56d09c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:34:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 504e80b1-8a90-4eed-9f55-d8faf573442b kind: Event lastTimestamp: "2026-07-28T17:34:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:46Z" name: success-200-isvc-d1fc0-predictor.18c6848c873cc4a8 namespace: kserve-ci-e2e-test resourceVersion: "22695" uid: 174fb5b2-8a2e-4124-98e7-943dd3e2eb55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:34:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1fc0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 504e80b1-8a90-4eed-9f55-d8faf573442b kind: Event lastTimestamp: "2026-07-28T17:34:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:46Z" name: success-200-isvc-d1fc0-predictor.18c6848c87a745b4 namespace: kserve-ci-e2e-test resourceVersion: "22696" uid: d91425e6-6a89-4052-8a69-19002bab59a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 3e0975eb-b394-4ae4-b09f-ae2735b0b564 kind: Event lastTimestamp: "2026-07-28T17:34:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:20Z" name: success-200-isvc-d1fc0.18c6848588f3ef43 namespace: kserve-ci-e2e-test resourceVersion: "22299" uid: 3ee0d7f6-2a18-4d10-8c9c-a7da0717de69 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 3e0975eb-b394-4ae4-b09f-ae2735b0b564 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-d1fc0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d1fc0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:31Z" name: success-200-isvc-d1fc0.18c6848589552875 namespace: kserve-ci-e2e-test resourceVersion: "21733" uid: a0506bd4-6dc7-4e0d-9b99-b39d0ab52926 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 3e0975eb-b394-4ae4-b09f-ae2735b0b564 kind: Event lastTimestamp: "2026-07-28T17:33:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d1fc0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:31Z" name: success-200-isvc-d1fc0.18c6848589557c37 namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: d55596b8-39ce-4172-bbb9-cf9376448b59 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1fc0 namespace: kserve-ci-e2e-test resourceVersion: "22297" uid: 3e0975eb-b394-4ae4-b09f-ae2735b0b564 kind: Event lastTimestamp: "2026-07-28T17:34:20Z" message: InferenceService [success-200-isvc-d1fc0] is Ready metadata: creationTimestamp: "2026-07-28T17:34:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:20Z" name: success-200-isvc-d1fc0.18c68490dd7aa4ab namespace: kserve-ci-e2e-test resourceVersion: "22298" uid: 1dbdecda-e523-497f-83a8-1b1f3e709b56 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:33:16.923258Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:33:16Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c684822c5385ac namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 63c15c13-76e9-4c5b-8993-bae7922f2ffe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ef873-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:17Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c6848234d04a9e namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 50f4c3b4-87a9-4851-8646-960dd07ebc1d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:33:17Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c6848269df5002 namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 81ff2ae6-bc1b-442d-aee4-8b61cceb180b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:33:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:17Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c684826b7daa38 namespace: kserve-ci-e2e-test resourceVersion: "21535" uid: f07f69d0-f16f-4884-a90b-693ea2b907ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:33:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:18Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c684826e74d7bc namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: 1f34fda1-cc6d-4037-a8b0-9f6b02f1b18c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:33:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:18Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c684826ee6564e namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: 58421ae5-e74f-4bb6-a10e-1f567035c23d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:33:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:18Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c684826ef19d36 namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: bdb102c4-c383-4d6c-979b-2794b57f9b56 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:33:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:18Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c6848271fe4048 namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: fe014677-64a2-4898-a657-5757d3325c84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:33:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:18Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c68482727158ba namespace: kserve-ci-e2e-test resourceVersion: "21540" uid: 38dc49dd-3c1d-4b73-973d-10dd5faa8c13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:33:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:33:54Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c684829fbc8502 namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: f05ec450-6a5a-4507-a6e6-52e799070d00 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c684928c2e58aa namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 36496d50-9fd8-4609-b304-3a23a7d88205 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:34:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:34:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:27Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c684928c2f1235 namespace: kserve-ci-e2e-test resourceVersion: "22419" uid: 4e0e8455-8a6e-46cd-b0ce-a3263d814a9b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 0715e64e-bb8d-4c48-b07d-724573ab36c0 kind: Event lastTimestamp: "2026-07-28T17:34:29Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:34:29Z" name: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4.18c68493282320d1 namespace: kserve-ci-e2e-test resourceVersion: "22507" uid: bcff71d4-8773-45cb-a5ee-f5f4a4a0aac0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ef873-predictor-69fd7d9b9 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: f85b8d8a-b018-4027-8c75-ec7b2e11545a kind: Event lastTimestamp: "2026-07-28T17:33:16Z" message: 'Created pod: success-200-isvc-ef873-predictor-69fd7d9b9-zhbb4' metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:16Z" name: success-200-isvc-ef873-predictor-69fd7d9b9.18c684822bbefabe namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: c336b8f3-e69b-403a-95cf-c0a620520234 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 40a002ce-b9ae-43db-9ad6-c7db6764b940 kind: Event lastTimestamp: "2026-07-28T17:33:16Z" message: Scaled up replica set success-200-isvc-ef873-predictor-69fd7d9b9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:16Z" name: success-200-isvc-ef873-predictor.18c684822a199c83 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: f8db308d-94e1-4efd-8757-04ca2ef54918 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: f0b5bf53-9528-47c0-a1c8-ce422aeae9fb kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:46Z" name: success-200-isvc-ef873-predictor.18c68485a8a452a1 namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: 01d3bfe1-4d26-4d9a-b59f-db215652cabc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:33:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: f0b5bf53-9528-47c0-a1c8-ce422aeae9fb kind: Event lastTimestamp: "2026-07-28T17:33:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:33:46Z" name: success-200-isvc-ef873-predictor.18c68485a915cdbb namespace: kserve-ci-e2e-test resourceVersion: "21965" uid: d9e9a8ef-5b46-41ca-85d1-b232871b5a4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:34:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: f0b5bf53-9528-47c0-a1c8-ce422aeae9fb kind: Event lastTimestamp: "2026-07-28T17:34:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:16Z" name: success-200-isvc-ef873-predictor.18c6848ca6a988d6 namespace: kserve-ci-e2e-test resourceVersion: "22232" uid: efa2ccb7-3f42-4148-8a9b-3915bd0c8e01 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:34:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ef873-predictor namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: f0b5bf53-9528-47c0-a1c8-ce422aeae9fb kind: Event lastTimestamp: "2026-07-28T17:34:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:34:16Z" name: success-200-isvc-ef873-predictor.18c6848ca72bad54 namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 807c2831-0ca9-4bb0-aa90-50285adea88f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ef873 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: 0d645759-8e7b-438e-910a-b1bcdfc855ec kind: Event lastTimestamp: "2026-07-28T17:34:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:05Z" name: success-200-isvc-ef873.18c684822a225743 namespace: kserve-ci-e2e-test resourceVersion: "22117" uid: ebfe1489-5b55-469a-8b75-e664d0b0d189 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ef873 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: 0d645759-8e7b-438e-910a-b1bcdfc855ec kind: Event lastTimestamp: "2026-07-28T17:33:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-ef873": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ef873": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:16Z" name: success-200-isvc-ef873.18c684822a936d4e namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 821da34d-ed00-49ff-825d-bb5a3be7c7fa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:33:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ef873 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: 0d645759-8e7b-438e-910a-b1bcdfc855ec kind: Event lastTimestamp: "2026-07-28T17:33:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ef873": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:33:16Z" name: success-200-isvc-ef873.18c684822a93c22f namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: ad34aa1c-aed4-43e0-b41e-61ba22b82de1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:34:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ef873 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: 0d645759-8e7b-438e-910a-b1bcdfc855ec kind: Event lastTimestamp: "2026-07-28T17:34:04Z" message: InferenceService [success-200-isvc-ef873] is Ready metadata: creationTimestamp: "2026-07-28T17:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:34:04Z" name: success-200-isvc-ef873.18c6848d5b9d2104 namespace: kserve-ci-e2e-test resourceVersion: "22116" uid: 6bac0022-5a3a-4415-801d-753996450622 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:44:42.224868Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0dc16-6959997c7c-wpt8k namespace: kserve-ci-e2e-test resourceVersion: "27815" uid: 8363b843-9b61-46b5-a905-f0e27057eda5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-0dc16-6959997c7c-wpt8k to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-07-28T17:44:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:44:42Z" name: switch-graph-0dc16-6959997c7c-wpt8k.18c68521bb7c33fc namespace: kserve-ci-e2e-test resourceVersion: "27820" uid: 382ca8a9-ebd4-40dc-b358-de4c3b6bce6a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:42Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0dc16-6959997c7c-wpt8k namespace: kserve-ci-e2e-test resourceVersion: "27818" uid: 8363b843-9b61-46b5-a905-f0e27057eda5 kind: Event lastTimestamp: "2026-07-28T17:44:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-0dc16-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:42Z" name: switch-graph-0dc16-6959997c7c-wpt8k.18c68521c3ecba0d namespace: kserve-ci-e2e-test resourceVersion: "27833" uid: cd67da62-55ad-4c23-9bc8-600c538b472e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:43Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0dc16-6959997c7c-wpt8k namespace: kserve-ci-e2e-test resourceVersion: "27823" uid: 8363b843-9b61-46b5-a905-f0e27057eda5 kind: Event lastTimestamp: "2026-07-28T17:44:43Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:44:43Z" name: switch-graph-0dc16-6959997c7c-wpt8k.18c68521f8d37195 namespace: kserve-ci-e2e-test resourceVersion: "27841" uid: d808495d-c58d-44eb-9100-1f985249e67c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dc16} kind: Pod name: switch-graph-0dc16-6959997c7c-wpt8k namespace: kserve-ci-e2e-test resourceVersion: "27818" uid: 8363b843-9b61-46b5-a905-f0e27057eda5 kind: Event lastTimestamp: "2026-07-28T17:44:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:43Z" name: switch-graph-0dc16-6959997c7c-wpt8k.18c68521fa1b366e namespace: kserve-ci-e2e-test resourceVersion: "27843" uid: 4dbd88c4-218d-4f2d-8cfc-f4f92885bcce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dc16} kind: Pod name: switch-graph-0dc16-6959997c7c-wpt8k namespace: kserve-ci-e2e-test resourceVersion: "27818" uid: 8363b843-9b61-46b5-a905-f0e27057eda5 kind: Event lastTimestamp: "2026-07-28T17:44:43Z" message: 'Created container: switch-graph-0dc16' metadata: creationTimestamp: "2026-07-28T17:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:43Z" name: switch-graph-0dc16-6959997c7c-wpt8k.18c68521fd02eaa0 namespace: kserve-ci-e2e-test resourceVersion: "27844" uid: 23385892-5934-40c2-8671-10dc6f8ac57a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dc16} kind: Pod name: switch-graph-0dc16-6959997c7c-wpt8k namespace: kserve-ci-e2e-test resourceVersion: "27818" uid: 8363b843-9b61-46b5-a905-f0e27057eda5 kind: Event lastTimestamp: "2026-07-28T17:44:43Z" message: Started container switch-graph-0dc16 metadata: creationTimestamp: "2026-07-28T17:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:44:43Z" name: switch-graph-0dc16-6959997c7c-wpt8k.18c68521fd6f94df namespace: kserve-ci-e2e-test resourceVersion: "27845" uid: 76ae552e-1d3d-46c5-ab0c-9f94371310cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:01:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dc16} kind: Pod name: switch-graph-0dc16-6959997c7c-wpt8k namespace: kserve-ci-e2e-test resourceVersion: "27818" uid: 8363b843-9b61-46b5-a905-f0e27057eda5 kind: Event lastTimestamp: "2026-07-28T18:01:01Z" message: Stopping container switch-graph-0dc16 metadata: creationTimestamp: "2026-07-28T18:01:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:01:01Z" name: switch-graph-0dc16-6959997c7c-wpt8k.18c68605bdd32e49 namespace: kserve-ci-e2e-test resourceVersion: "34705" uid: 729f13e3-6caa-4e1c-af8c-00c322aaa245 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T18:01:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dc16} kind: Pod name: switch-graph-0dc16-6959997c7c-wpt8k namespace: kserve-ci-e2e-test resourceVersion: "27818" uid: 8363b843-9b61-46b5-a905-f0e27057eda5 kind: Event lastTimestamp: "2026-07-28T18:01:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T18:01:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:01:25Z" name: switch-graph-0dc16-6959997c7c-wpt8k.18c6860691795d87 namespace: kserve-ci-e2e-test resourceVersion: "34947" uid: de02d87e-0b5e-44aa-b3d2-57f90ee2c500 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-0dc16-6959997c7c namespace: kserve-ci-e2e-test resourceVersion: "27811" uid: e973b96a-e412-46fd-97dc-a07d9dad2131 kind: Event lastTimestamp: "2026-07-28T17:44:42Z" message: 'Created pod: switch-graph-0dc16-6959997c7c-wpt8k' metadata: creationTimestamp: "2026-07-28T17:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:42Z" name: switch-graph-0dc16-6959997c7c.18c68521bb054b3d namespace: kserve-ci-e2e-test resourceVersion: "27817" uid: e9f34d3e-21b0-42b9-84a8-c4a109a5f759 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "27808" uid: 2181f2dc-c1ab-4e1a-9003-fb92b1259539 kind: Event lastTimestamp: "2026-07-28T17:44:42Z" message: Scaled up replica set switch-graph-0dc16-6959997c7c from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:44:42Z" name: switch-graph-0dc16.18c68521b9f41776 namespace: kserve-ci-e2e-test resourceVersion: "27813" uid: 7ac5b75a-b323-44e8-8e82-964c176cf6db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:44:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "27826" uid: 689e1408-2d06-47e3-ad07-21a3f79e06d8 kind: Event lastTimestamp: "2026-07-28T17:44:42Z" message: InferenceGraph [switch-graph-0dc16] is Ready metadata: creationTimestamp: "2026-07-28T17:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:44:42Z" name: switch-graph-0dc16.18c68521be09325e namespace: kserve-ci-e2e-test resourceVersion: "27828" uid: 2aa4e7df-187c-4618-85cd-4297deac2ab5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "27810" uid: 9ebbbf84-3870-4d6c-804f-983d58e8010b kind: Event lastTimestamp: "2026-07-28T17:45:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:12Z" name: switch-graph-0dc16.18c6852538937901 namespace: kserve-ci-e2e-test resourceVersion: "28127" uid: 9703b8f5-3695-41af-ba7d-482deb205558 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:44:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0dc16 namespace: kserve-ci-e2e-test resourceVersion: "27810" uid: 9ebbbf84-3870-4d6c-804f-983d58e8010b kind: Event lastTimestamp: "2026-07-28T17:45:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:45:12Z" name: switch-graph-0dc16.18c6852538e89254 namespace: kserve-ci-e2e-test resourceVersion: "28128" uid: 32bb33e5-a7e1-438e-84f3-622fc22db746 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:24:02.130224Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-391b7-6f8f74c7b5-hsztp namespace: kserve-ci-e2e-test resourceVersion: "17173" uid: 1740b307-ce88-4b2b-9ca1-811b2f70144a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-391b7-6f8f74c7b5-hsztp to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:24:02Z" name: switch-graph-391b7-6f8f74c7b5-hsztp.18c684010016563c namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: 6a3bb84e-e4f4-4a48-9efd-33ba47cf517b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-391b7-6f8f74c7b5-hsztp namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: 1740b307-ce88-4b2b-9ca1-811b2f70144a kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-391b7-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:02Z" name: switch-graph-391b7-6f8f74c7b5-hsztp.18c6840108666848 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: aa06f574-5897-4ffd-870f-2f64112f5168 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-391b7-6f8f74c7b5-hsztp namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 1740b307-ce88-4b2b-9ca1-811b2f70144a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:24:03Z" name: switch-graph-391b7-6f8f74c7b5-hsztp.18c684013d99d7a1 namespace: kserve-ci-e2e-test resourceVersion: "17241" uid: 34bb861f-3091-4c62-b320-8090fe243587 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-391b7} kind: Pod name: switch-graph-391b7-6f8f74c7b5-hsztp namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: 1740b307-ce88-4b2b-9ca1-811b2f70144a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" already present on machine metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:03Z" name: switch-graph-391b7-6f8f74c7b5-hsztp.18c684013efe723c namespace: kserve-ci-e2e-test resourceVersion: "17244" uid: 758d01bd-e6ec-472e-bb83-9b91b09e4d33 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-391b7} kind: Pod name: switch-graph-391b7-6f8f74c7b5-hsztp namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: 1740b307-ce88-4b2b-9ca1-811b2f70144a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: 'Created container: switch-graph-391b7' metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:03Z" name: switch-graph-391b7-6f8f74c7b5-hsztp.18c6840141c2377e namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: b4f36e3a-473e-49dc-aa96-f4953bd98077 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-391b7} kind: Pod name: switch-graph-391b7-6f8f74c7b5-hsztp namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: 1740b307-ce88-4b2b-9ca1-811b2f70144a kind: Event lastTimestamp: "2026-07-28T17:24:03Z" message: Started container switch-graph-391b7 metadata: creationTimestamp: "2026-07-28T17:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:24:03Z" name: switch-graph-391b7-6f8f74c7b5-hsztp.18c68401422e6def namespace: kserve-ci-e2e-test resourceVersion: "17247" uid: 2f54c1d2-7c86-4b2a-ab0c-858360e68e09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-391b7} kind: Pod name: switch-graph-391b7-6f8f74c7b5-hsztp namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: 1740b307-ce88-4b2b-9ca1-811b2f70144a kind: Event lastTimestamp: "2026-07-28T17:32:16Z" message: Stopping container switch-graph-391b7 metadata: creationTimestamp: "2026-07-28T17:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:16Z" name: switch-graph-391b7-6f8f74c7b5-hsztp.18c68474302ea369 namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 813fd4fb-c384-471d-bf75-303611a57e14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-391b7} kind: Pod name: switch-graph-391b7-6f8f74c7b5-hsztp namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: 1740b307-ce88-4b2b-9ca1-811b2f70144a kind: Event lastTimestamp: "2026-07-28T17:32:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:32:44Z" name: switch-graph-391b7-6f8f74c7b5-hsztp.18c68474df927353 namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: dc88385a-d147-4dcd-bf47-2c1664c2db6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-391b7-6f8f74c7b5 namespace: kserve-ci-e2e-test resourceVersion: "17161" uid: 529ae382-c659-4bde-86a6-ee88c0372371 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: 'Created pod: switch-graph-391b7-6f8f74c7b5-hsztp' metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:02Z" name: switch-graph-391b7-6f8f74c7b5.18c68400ffc7afcb namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 7f21b2ec-9f1a-44f8-8815-24ab2c40497f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-391b7 namespace: kserve-ci-e2e-test resourceVersion: "17153" uid: 23e61507-3c65-474e-8261-bf6e5dc65495 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: Scaled up replica set switch-graph-391b7-6f8f74c7b5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:02Z" name: switch-graph-391b7.18c68400feb1da21 namespace: kserve-ci-e2e-test resourceVersion: "17169" uid: 7032bd52-70f5-4610-99cc-20961739aa17 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:24:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-391b7 namespace: kserve-ci-e2e-test resourceVersion: "17199" uid: c2caba25-2fa6-4db2-98fa-7c7436d87196 kind: Event lastTimestamp: "2026-07-28T17:24:02Z" message: InferenceGraph [switch-graph-391b7] is Ready metadata: creationTimestamp: "2026-07-28T17:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:24:02Z" name: switch-graph-391b7.18c6840104701212 namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: 8d782fc7-80a2-400b-8e77-f10eb27d396e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:24:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-391b7 namespace: kserve-ci-e2e-test resourceVersion: "17158" uid: 72aa8194-0e98-47d5-a984-2f7f8cbfdf92 kind: Event lastTimestamp: "2026-07-28T17:24:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:24:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:47Z" name: switch-graph-391b7.18c684047e0d21a8 namespace: kserve-ci-e2e-test resourceVersion: "17629" uid: 4f684865-996d-496b-9191-6c735554f995 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:24:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-391b7 namespace: kserve-ci-e2e-test resourceVersion: "17158" uid: 72aa8194-0e98-47d5-a984-2f7f8cbfdf92 kind: Event lastTimestamp: "2026-07-28T17:24:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:24:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:24:47Z" name: switch-graph-391b7.18c684047ebf0bea namespace: kserve-ci-e2e-test resourceVersion: "17630" uid: 9ab639d2-fca8-4040-bfd9-e7397e6e3aca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:22:51.676236Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b94d0-647d855485-ztlwt namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: e41d297a-7594-472f-8af9-8404ff789851 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-b94d0-647d855485-ztlwt to ip-10-0-129-82.ec2.internal metadata: creationTimestamp: "2026-07-28T17:22:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:22:51Z" name: switch-graph-b94d0-647d855485-ztlwt.18c683f098b31ee9 namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 13163f81-4cd5-40a2-a095-5a705ecbbf41 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-d86bf774c-2vdz5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b94d0-647d855485-ztlwt namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: e41d297a-7594-472f-8af9-8404ff789851 kind: Event lastTimestamp: "2026-07-28T17:22:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-b94d0-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:51Z" name: switch-graph-b94d0-647d855485-ztlwt.18c683f0a0f58ac6 namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: 90f78c40-d250-4996-8e7d-f1db1e7f6729 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:52Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b94d0-647d855485-ztlwt namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: e41d297a-7594-472f-8af9-8404ff789851 kind: Event lastTimestamp: "2026-07-28T17:22:52Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:22:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:22:52Z" name: switch-graph-b94d0-647d855485-ztlwt.18c683f0d5c04021 namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 805ffab6-7698-4489-98d7-fa8edb249980 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b94d0} kind: Pod name: switch-graph-b94d0-647d855485-ztlwt namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: e41d297a-7594-472f-8af9-8404ff789851 kind: Event lastTimestamp: "2026-07-28T17:22:52Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" metadata: creationTimestamp: "2026-07-28T17:22:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:52Z" name: switch-graph-b94d0-647d855485-ztlwt.18c683f0d72a0d55 namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 8426a795-668e-4ff4-998d-055d04f10dc3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b94d0} kind: Pod name: switch-graph-b94d0-647d855485-ztlwt namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: e41d297a-7594-472f-8af9-8404ff789851 kind: Event lastTimestamp: "2026-07-28T17:22:54Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:2d78b1e75ce4a3e633713c1f964bde1c2425164638b26ad77af20d6340c844d7" in 1.732s (1.732s including waiting). Image size: 218226779 bytes.' metadata: creationTimestamp: "2026-07-28T17:22:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:54Z" name: switch-graph-b94d0-647d855485-ztlwt.18c683f13e6d7c00 namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 93bc963e-b6e0-41bf-9ecd-c8effef44d1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b94d0} kind: Pod name: switch-graph-b94d0-647d855485-ztlwt namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: e41d297a-7594-472f-8af9-8404ff789851 kind: Event lastTimestamp: "2026-07-28T17:22:54Z" message: 'Created container: switch-graph-b94d0' metadata: creationTimestamp: "2026-07-28T17:22:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:54Z" name: switch-graph-b94d0-647d855485-ztlwt.18c683f1410cce10 namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: 6aa218ce-90a1-44a1-8ef9-852cde387c65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b94d0} kind: Pod name: switch-graph-b94d0-647d855485-ztlwt namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: e41d297a-7594-472f-8af9-8404ff789851 kind: Event lastTimestamp: "2026-07-28T17:22:54Z" message: Started container switch-graph-b94d0 metadata: creationTimestamp: "2026-07-28T17:22:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:22:54Z" name: switch-graph-b94d0-647d855485-ztlwt.18c683f1416b2b6e namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 8c257bfe-e293-428e-8e29-278444a83e3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b94d0} kind: Pod name: switch-graph-b94d0-647d855485-ztlwt namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: e41d297a-7594-472f-8af9-8404ff789851 kind: Event lastTimestamp: "2026-07-28T17:23:01Z" message: Stopping container switch-graph-b94d0 metadata: creationTimestamp: "2026-07-28T17:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:01Z" name: switch-graph-b94d0-647d855485-ztlwt.18c683f2f7dbf37c namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 8364fac0-999b-46aa-a985-2e42a0827853 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b94d0} kind: Pod name: switch-graph-b94d0-647d855485-ztlwt namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: e41d297a-7594-472f-8af9-8404ff789851 kind: Event lastTimestamp: "2026-07-28T17:23:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:23:30Z" name: switch-graph-b94d0-647d855485-ztlwt.18c683f3d2c6d6ae namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: d5db3292-9c86-4261-b504-b7c9a690c1a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-82.ec2.internal source: component: kubelet host: ip-10-0-129-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-b94d0-647d855485 namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: ed2ceaba-bddd-42b0-bd5c-7cd45711d8b3 kind: Event lastTimestamp: "2026-07-28T17:22:51Z" message: 'Created pod: switch-graph-b94d0-647d855485-ztlwt' metadata: creationTimestamp: "2026-07-28T17:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:51Z" name: switch-graph-b94d0-647d855485.18c683f098412506 namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: f961befa-87d4-4214-b618-973ce281b2e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-b94d0 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: f0e3d4e7-8f6c-418b-be35-29f7c2b78230 kind: Event lastTimestamp: "2026-07-28T17:22:51Z" message: Scaled up replica set switch-graph-b94d0-647d855485 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:22:51Z" name: switch-graph-b94d0.18c683f097621458 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 008f8e28-dbbd-4bd7-892b-22f28aabbb25 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:22:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b94d0 namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: 0f17cc52-ca11-4b45-9b28-731fde8a6eb0 kind: Event lastTimestamp: "2026-07-28T17:22:51Z" message: InferenceGraph [switch-graph-b94d0] is Ready metadata: creationTimestamp: "2026-07-28T17:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:22:51Z" name: switch-graph-b94d0.18c683f09b507724 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 0dabd939-9aa2-43a5-891b-5f8234dfb662 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "34978"