--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:26:39.606699Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cb683-6bd9cdd588-hncbs namespace: kserve-ci-e2e-test resourceVersion: "21495" uid: 84913fee-eaea-4b13-b8df-d0c70596ad75 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-cb683-6bd9cdd588-hncbs to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-07-27T21:26:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:26:39Z" name: ensemble-graph-cb683-6bd9cdd588-hncbs.18c642a9dbfd7dc8 namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: 291d7303-c619-4701-94b3-fe4a91c29bd9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:40Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cb683-6bd9cdd588-hncbs namespace: kserve-ci-e2e-test resourceVersion: "21505" uid: 84913fee-eaea-4b13-b8df-d0c70596ad75 kind: Event lastTimestamp: "2026-07-27T21:26:40Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:26:40Z" name: ensemble-graph-cb683-6bd9cdd588-hncbs.18c642a9f5e1fef7 namespace: kserve-ci-e2e-test resourceVersion: "21520" uid: e3216f6a-9838-43a8-90d6-a3ce8b09a7db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cb683} kind: Pod name: ensemble-graph-cb683-6bd9cdd588-hncbs namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: 84913fee-eaea-4b13-b8df-d0c70596ad75 kind: Event lastTimestamp: "2026-07-27T21:26:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-27T21:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:40Z" name: ensemble-graph-cb683-6bd9cdd588-hncbs.18c642a9f73e671d namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 1a915caa-cc98-4f69-94a2-0ccb2eabc661 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cb683} kind: Pod name: ensemble-graph-cb683-6bd9cdd588-hncbs namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: 84913fee-eaea-4b13-b8df-d0c70596ad75 kind: Event lastTimestamp: "2026-07-27T21:26:40Z" message: 'Created container: ensemble-graph-cb683' metadata: creationTimestamp: "2026-07-27T21:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:40Z" name: ensemble-graph-cb683-6bd9cdd588-hncbs.18c642a9fa2106fb namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: 417bc2fa-fcd6-446f-ad67-6f1eef60177f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cb683} kind: Pod name: ensemble-graph-cb683-6bd9cdd588-hncbs namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: 84913fee-eaea-4b13-b8df-d0c70596ad75 kind: Event lastTimestamp: "2026-07-27T21:26:40Z" message: Started container ensemble-graph-cb683 metadata: creationTimestamp: "2026-07-27T21:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:40Z" name: ensemble-graph-cb683-6bd9cdd588-hncbs.18c642a9fa9710d8 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: 740e2b54-220d-4f4f-ab6a-e40fb59fd470 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cb683} kind: Pod name: ensemble-graph-cb683-6bd9cdd588-hncbs namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: 84913fee-eaea-4b13-b8df-d0c70596ad75 kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: Stopping container ensemble-graph-cb683 metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:49Z" name: ensemble-graph-cb683-6bd9cdd588-hncbs.18c642ac343e4551 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: 31dda12c-c29e-4936-87dc-4da94d3ae61d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cb683} kind: Pod name: ensemble-graph-cb683-6bd9cdd588-hncbs namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: 84913fee-eaea-4b13-b8df-d0c70596ad75 kind: Event lastTimestamp: "2026-07-27T21:27:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:16Z" name: ensemble-graph-cb683-6bd9cdd588-hncbs.18c642aca90637b8 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: 53d45b44-c8cb-4454-bf55-3a41fd5b93df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-cb683-6bd9cdd588 namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: e6059feb-aa8e-46af-8155-1999004ebcc4 kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: 'Created pod: ensemble-graph-cb683-6bd9cdd588-hncbs' metadata: creationTimestamp: "2026-07-27T21:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: ensemble-graph-cb683-6bd9cdd588.18c642a9db9f8130 namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: db181864-e4af-4dce-a17e-cf7dbb14dfd6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-cb683 namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 73a5c6e2-c1dc-434b-81b1-6716ccc18fd6 kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: Scaled up replica set ensemble-graph-cb683-6bd9cdd588 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: ensemble-graph-cb683.18c642a9daaaac31 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 3c791c9e-a61e-4e94-8144-1bdb190113c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-cb683 namespace: kserve-ci-e2e-test resourceVersion: "21509" uid: 13de90ef-6c1a-494c-9b76-bb0de357eb72 kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: InferenceGraph [ensemble-graph-cb683] is Ready metadata: creationTimestamp: "2026-07-27T21:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:39Z" name: ensemble-graph-cb683.18c642a9de890a80 namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 47801998-2319-4dc1-bab3-b8daac1f5076 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:27:49.927841Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e26c5-5545679b85-cl99w namespace: kserve-ci-e2e-test resourceVersion: "22460" uid: 8cb225b5-989f-44f8-907e-ef72ca380972 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-e26c5-5545679b85-cl99w to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:27:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:27:49Z" name: ensemble-graph-e26c5-5545679b85-cl99w.18c642ba3b74ee30 namespace: kserve-ci-e2e-test resourceVersion: "22466" uid: 80a4cc94-e8ac-4cf6-97aa-18585c400ca1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:50Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e26c5-5545679b85-cl99w namespace: kserve-ci-e2e-test resourceVersion: "22470" uid: 8cb225b5-989f-44f8-907e-ef72ca380972 kind: Event lastTimestamp: "2026-07-27T21:27:50Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:27:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:27:50Z" name: ensemble-graph-e26c5-5545679b85-cl99w.18c642ba550ec903 namespace: kserve-ci-e2e-test resourceVersion: "22483" uid: e99352d0-3d74-4fa0-8b9f-40b4f1c63b1d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e26c5} kind: Pod name: ensemble-graph-e26c5-5545679b85-cl99w namespace: kserve-ci-e2e-test resourceVersion: "22463" uid: 8cb225b5-989f-44f8-907e-ef72ca380972 kind: Event lastTimestamp: "2026-07-27T21:27:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-27T21:27:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:50Z" name: ensemble-graph-e26c5-5545679b85-cl99w.18c642ba56440e34 namespace: kserve-ci-e2e-test resourceVersion: "22485" uid: 51972ea1-4af7-437c-b12e-ac813209b349 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e26c5} kind: Pod name: ensemble-graph-e26c5-5545679b85-cl99w namespace: kserve-ci-e2e-test resourceVersion: "22463" uid: 8cb225b5-989f-44f8-907e-ef72ca380972 kind: Event lastTimestamp: "2026-07-27T21:27:50Z" message: 'Created container: ensemble-graph-e26c5' metadata: creationTimestamp: "2026-07-27T21:27:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:50Z" name: ensemble-graph-e26c5-5545679b85-cl99w.18c642ba591aaad6 namespace: kserve-ci-e2e-test resourceVersion: "22486" uid: 9298c3e3-777f-458f-88e1-6c36a3013d5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e26c5} kind: Pod name: ensemble-graph-e26c5-5545679b85-cl99w namespace: kserve-ci-e2e-test resourceVersion: "22463" uid: 8cb225b5-989f-44f8-907e-ef72ca380972 kind: Event lastTimestamp: "2026-07-27T21:27:50Z" message: Started container ensemble-graph-e26c5 metadata: creationTimestamp: "2026-07-27T21:27:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:50Z" name: ensemble-graph-e26c5-5545679b85-cl99w.18c642ba598e2ec5 namespace: kserve-ci-e2e-test resourceVersion: "22487" uid: 8255e4a6-ee24-49e1-84c5-e268abfd650d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e26c5} kind: Pod name: ensemble-graph-e26c5-5545679b85-cl99w namespace: kserve-ci-e2e-test resourceVersion: "22463" uid: 8cb225b5-989f-44f8-907e-ef72ca380972 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: Stopping container ensemble-graph-e26c5 metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:04Z" name: ensemble-graph-e26c5-5545679b85-cl99w.18c6432d623a3c37 namespace: kserve-ci-e2e-test resourceVersion: "26252" uid: 50616d5f-6372-4708-bc8f-911bb09946a7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:36:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e26c5} kind: Pod name: ensemble-graph-e26c5-5545679b85-cl99w namespace: kserve-ci-e2e-test resourceVersion: "22463" uid: 8cb225b5-989f-44f8-907e-ef72ca380972 kind: Event lastTimestamp: "2026-07-27T21:36:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:31Z" name: ensemble-graph-e26c5-5545679b85-cl99w.18c6432ddd929377 namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: 78c38309-612b-41ac-be8d-315ffc9edac2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-e26c5-5545679b85 namespace: kserve-ci-e2e-test resourceVersion: "22457" uid: 01933e5d-7198-4f66-80e9-8c8f05eb0dbb kind: Event lastTimestamp: "2026-07-27T21:27:49Z" message: 'Created pod: ensemble-graph-e26c5-5545679b85-cl99w' metadata: creationTimestamp: "2026-07-27T21:27:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:49Z" name: ensemble-graph-e26c5-5545679b85.18c642ba3ac955e3 namespace: kserve-ci-e2e-test resourceVersion: "22462" uid: e109cac3-428e-4b05-b03e-5407759df2d0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "22454" uid: 2cc2b934-ca83-4f00-875a-7da02bd605cd kind: Event lastTimestamp: "2026-07-27T21:27:49Z" message: Scaled up replica set ensemble-graph-e26c5-5545679b85 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:27:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:49Z" name: ensemble-graph-e26c5.18c642ba39de846f namespace: kserve-ci-e2e-test resourceVersion: "22458" uid: 8567810d-d511-4db0-a9b3-469a18518476 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "22472" uid: 8d0f99cb-b044-406c-9694-94f2e19822ae kind: Event lastTimestamp: "2026-07-27T21:27:49Z" message: InferenceGraph [ensemble-graph-e26c5] is Ready metadata: creationTimestamp: "2026-07-27T21:27:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:27:49Z" name: ensemble-graph-e26c5.18c642ba3f495a48 namespace: kserve-ci-e2e-test resourceVersion: "22475" uid: a4cb8829-5ccd-4cf3-885c-258d0701f8c7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:28:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "22456" uid: 71d73236-d184-400a-bd58-ca593f071dd2 kind: Event lastTimestamp: "2026-07-27T21:28:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:19Z" name: ensemble-graph-e26c5.18c642bdb839e9c1 namespace: kserve-ci-e2e-test resourceVersion: "22761" uid: 7eb2ca1b-0612-425c-91a9-50356473fe02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:28:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "22456" uid: 71d73236-d184-400a-bd58-ca593f071dd2 kind: Event lastTimestamp: "2026-07-27T21:28:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:19Z" name: ensemble-graph-e26c5.18c642bdb89aee58 namespace: kserve-ci-e2e-test resourceVersion: "22762" uid: f7cad693-dba1-4b37-9690-efedda983888 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:14:34.713874Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-070ff-predictor-59657fff56-glz8v to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:14:34Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c64201150307b9 namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: bad49faa-c3e6-4dc3-98bf-aa318bd053df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:14:35Z" message: Add eth0 [10.133.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:14:35Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c642012fc3682e namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 041b685d-9c8a-4d04-820c-3bd3e2cc4b25 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:14:35Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-27T21:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:35Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c64201313498ed namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: bc040027-b188-4eb0-a1fa-421f0ef42048 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:14:47Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" in 11.899s (11.899s including waiting). Image size: 1333940844 bytes.' metadata: creationTimestamp: "2026-07-27T21:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:47Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c64203f67b615a namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: da968fca-1d00-47f5-aeae-993e55f56ebb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:14:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:47Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c64203f973beef namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: 863a065d-f8b5-41e3-9c19-d5773a0e0ee2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:14:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:47Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c64203f9eb4709 namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: a9e52115-67e6-4264-b08e-dce061805477 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:14:47Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-27T21:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:47Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c64203f9f80b98 namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: 4b10da81-1964-4055-8086-2f4949803008 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:14:49Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.24s (2.24s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:49Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c642047f85eec4 namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 18e496bc-5655-46ca-8508-c5814d48b324 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:14:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:49Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c6420482a5ef74 namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: d9e15b61-6f4d-4cce-af1d-47d155218577 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:14:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:49Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c64204831e48c6 namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: a4364a65-89cd-4994-87a0-02e6c449cda8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c642049d7abb00 namespace: kserve-ci-e2e-test resourceVersion: "16442" uid: d2b5a459-7549-4fde-b533-c15a67fc76c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:24Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c6421ab44bfacc namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: bf2c00c6-8cfd-44bb-a067-229ec297dfb3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:24Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c6421ab44c5c19 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 302fcb49-05d6-4592-9c1d-fd2faf8063c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-070ff-predictor-59657fff56-glz8v namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 9a2b84e8-0b58-4151-96e6-467b6b90d0bb kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: 'Readiness probe failed: Get "https://10.133.0.16:8643/healthz": dial tcp 10.133.0.16:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: error-404-isvc-070ff-predictor-59657fff56-glz8v.18c6421af7bc49a9 namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 93f5d271-4b79-43ca-80cb-f13bc0b825ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-070ff-predictor-59657fff56 namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: 1d1cfe72-63b5-40ea-825a-0a852d5f5b97 kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: 'Created pod: error-404-isvc-070ff-predictor-59657fff56-glz8v' metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:34Z" name: error-404-isvc-070ff-predictor-59657fff56.18c6420113baeef7 namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: f45e2f72-d2ba-4f5b-9542-613d24fa9975 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 3e282689-10f7-4616-8d5d-21220c814d16 kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: Scaled up replica set error-404-isvc-070ff-predictor-59657fff56 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:34Z" name: error-404-isvc-070ff-predictor.18c6420111dd89bf namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 7265650c-788f-4aa3-b4a1-cb4264f69e00 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: cdf3e93d-77ce-460b-8c9b-819aa4d3616a kind: Event lastTimestamp: "2026-07-27T21:15:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:19Z" name: error-404-isvc-070ff-predictor.18c64204907be0ab namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: 39a66bc2-0d2e-4fc5-8f24-da3025fcf9f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: cdf3e93d-77ce-460b-8c9b-819aa4d3616a kind: Event lastTimestamp: "2026-07-27T21:15:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:19Z" name: error-404-isvc-070ff-predictor.18c6420490f56e7b namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 2fe88c62-6600-4dbe-88db-470d345d87a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:15:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: cdf3e93d-77ce-460b-8c9b-819aa4d3616a kind: Event lastTimestamp: "2026-07-27T21:16:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:15:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:04Z" name: error-404-isvc-070ff-predictor.18c6420f0cfcd727 namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: af0f381c-3d85-485f-a307-4045370732de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:15:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: cdf3e93d-77ce-460b-8c9b-819aa4d3616a kind: Event lastTimestamp: "2026-07-27T21:16:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:15:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:04Z" name: error-404-isvc-070ff-predictor.18c6420f0d61d238 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 3329d688-d2fc-4957-b33c-3c0bd83db1f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-070ff namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 647749e7-13ee-41d0-a025-54c740f32179 kind: Event lastTimestamp: "2026-07-27T21:15:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:15:36Z" name: error-404-isvc-070ff.18c6420111eae4f2 namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: f889e798-a65c-484d-822e-99248ac8deee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:15:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-070ff namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 647749e7-13ee-41d0-a025-54c740f32179 kind: Event lastTimestamp: "2026-07-27T21:15:35Z" message: InferenceService [error-404-isvc-070ff] is Ready metadata: creationTimestamp: "2026-07-27T21:15:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:15:35Z" name: error-404-isvc-070ff.18c6420f5862942a namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: 9eda72ab-689d-4962-9432-7f68f9b944a7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:26:09.392323Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0a095-predictor-745d9f4d88-hw9ff to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:26:09Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a2d312b54a namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: 0d004d41-2d77-4365-9010-6c1a0f702e28 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0a095-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:09Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a2df09cd5f namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: afb057cb-f23f-4575-be67-a6c309acf2c6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:26:10Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a310729633 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 4fc98ad4-7ae3-49ab-894c-0a847b0ac176 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a311eb37ee namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 24b00e30-af89-4b21-8532-4f35bddb0c2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a3151feff2 namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: aee11ee4-2797-4061-9f75-6456a1741fce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a3159c99b6 namespace: kserve-ci-e2e-test resourceVersion: "21209" uid: 1081c560-03d2-4c96-994e-35bebdd826d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a315a68f8c namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: c8f9158a-b968-46d4-a8bb-18a18c47673a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a3186681ad namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: c790b2d2-3f57-486b-9610-86a341ec446d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a318dcbacd namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: 7d299495-e27a-42cd-8e5d-f985f36c044f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:26:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:47Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642a33f4eb0bd namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: b54a3e06-36d0-4f10-a22c-04ab4ae1a1ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:19Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642b32937a803 namespace: kserve-ci-e2e-test resourceVersion: "22100" uid: 480ea997-f63c-498f-8f35-ef6b05fdb492 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:19Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642b329382a5e namespace: kserve-ci-e2e-test resourceVersion: "22104" uid: 59fe7162-5205-4b78-9f45-7b2779807832 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dcde100a-1025-4ee1-a150-487a1086f4d3 kind: Event lastTimestamp: "2026-07-27T21:27:22Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:22Z" name: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff.18c642b3c7672d9a namespace: kserve-ci-e2e-test resourceVersion: "22203" uid: cc075d0f-8b70-40a5-9c3b-635bad4a256a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0a095-predictor-745d9f4d88 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: a04658ad-2f2d-4294-8e1c-8a9ed804b8dc kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: 'Created pod: error-404-isvc-0a095-predictor-745d9f4d88-hw9ff' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:09Z" name: error-404-isvc-0a095-predictor-745d9f4d88.18c642a2d2aacd91 namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: e8ba41ef-f014-4c80-802a-81d890b1a4c2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 7cdddb08-e9a2-4b18-9a93-75805272863d kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: Scaled up replica set error-404-isvc-0a095-predictor-745d9f4d88 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:09Z" name: error-404-isvc-0a095-predictor.18c642a2d115c2cc namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 410de2db-1e4e-4c79-8090-24662e2b5c19 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 9ff610bb-925d-47d7-9c96-5bac6500e628 kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: error-404-isvc-0a095-predictor.18c642a64f8f16d6 namespace: kserve-ci-e2e-test resourceVersion: "21478" uid: b51653ee-6572-4787-b6a6-dce0d1954316 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 9ff610bb-925d-47d7-9c96-5bac6500e628 kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: error-404-isvc-0a095-predictor.18c642a64fea8668 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 5c2a10d3-ee8e-4987-bae4-4342784f4dc0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:26:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 9ff610bb-925d-47d7-9c96-5bac6500e628 kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:26:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:09Z" name: error-404-isvc-0a095-predictor.18c642ad4d44ed9f namespace: kserve-ci-e2e-test resourceVersion: "21944" uid: ab60920b-105e-4ecd-be6e-571f827b4f55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:26:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 9ff610bb-925d-47d7-9c96-5bac6500e628 kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:26:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:09Z" name: error-404-isvc-0a095-predictor.18c642ad4da5bc89 namespace: kserve-ci-e2e-test resourceVersion: "21945" uid: f43a459d-cbdd-454d-a756-c7b8fc792e5a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 524c300b-554b-4f02-ab4e-a0d1584e095f kind: Event lastTimestamp: "2026-07-27T21:26:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:57Z" name: error-404-isvc-0a095.18c642a2d126904d namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: 552e80d7-b0a8-45ee-ade0-1d31abe3b05b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 524c300b-554b-4f02-ab4e-a0d1584e095f kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-0a095": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0a095": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:09Z" name: error-404-isvc-0a095.18c642a2d18b1c03 namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: e34a2ebd-c73a-4bf1-919c-ce42d10df7c3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 524c300b-554b-4f02-ab4e-a0d1584e095f kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0a095": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:09Z" name: error-404-isvc-0a095.18c642a2d18b740b namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 3e514e3c-c58f-49a8-9261-271604047a01 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: 524c300b-554b-4f02-ab4e-a0d1584e095f kind: Event lastTimestamp: "2026-07-27T21:26:57Z" message: InferenceService [error-404-isvc-0a095] is Ready metadata: creationTimestamp: "2026-07-27T21:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:57Z" name: error-404-isvc-0a095.18c642adfad5e6e8 namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: 54c16784-71f8-493f-a148-5bf03e0aca81 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:27:19.670421Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22130" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:27:19Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c642b32ff96261 namespace: kserve-ci-e2e-test resourceVersion: "22134" uid: c167b078-0f66-4112-8f81-fed6ce94a8e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:27:20Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c642b349b88493 namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: 64163021-9175-4206-b826-9d2eba6cfe48 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c642b34b1e74fd namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: 6fef1405-27eb-4c44-9ba1-07a9b7584e39 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c642b34db09e1f namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: 55e0ab92-3145-433a-a9a1-313c0c11db92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c642b34e22d00a namespace: kserve-ci-e2e-test resourceVersion: "22166" uid: cbf6e891-ab6e-4c8e-ba2f-ead3811b78d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c642b34e2e05b5 namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: 2d54f662-7e6e-4990-ab9e-0565da73cfe3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c642b350cb6950 namespace: kserve-ci-e2e-test resourceVersion: "22168" uid: 203b118b-5e0c-4982-81a5-c5010b042faf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c642b35153bc9d namespace: kserve-ci-e2e-test resourceVersion: "22169" uid: b4bb35fe-d989-4bf1-b588-e66adf67979a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-27T21:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:36:37Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:37Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c642b39a4c06f4 namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 48e84dbf-c7cb-4ba4-ad60-c45dd7c35ea7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:34Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c643345664f54d namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: b334bd7a-0948-474e-9b19-00cfb3a9f810 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:34Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c64334566608d2 namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: 01cd8261-0599-41c8-848c-5d3225f74262 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: eba7c1dc-88e9-4f1a-83ea-adb3be7f8d6d kind: Event lastTimestamp: "2026-07-27T21:36:37Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:37Z" name: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq.18c643350e9b2cae namespace: kserve-ci-e2e-test resourceVersion: "26788" uid: e18e203a-9851-40df-941f-d0f516f9b140 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0f5d7-predictor-655fb59d6 namespace: kserve-ci-e2e-test resourceVersion: "22123" uid: 9a2fc90e-3162-47cb-9377-404f378be813 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: 'Created pod: error-404-isvc-0f5d7-predictor-655fb59d6-xv9mq' metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:19Z" name: error-404-isvc-0f5d7-predictor-655fb59d6.18c642b32f953401 namespace: kserve-ci-e2e-test resourceVersion: "22132" uid: 8ae44320-9534-448e-be7b-fd7609312862 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22119" uid: 26f9d6c7-01a1-46bf-8f53-3d76bec730d4 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: Scaled up replica set error-404-isvc-0f5d7-predictor-655fb59d6 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:19Z" name: error-404-isvc-0f5d7-predictor.18c642b32ddce68e namespace: kserve-ci-e2e-test resourceVersion: "22124" uid: e05ef22a-fe72-4ff8-a162-98f7652857fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22121" uid: 6e43f071-5c26-4fb5-997e-8d9a8b3778ea kind: Event lastTimestamp: "2026-07-27T21:27:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:49Z" name: error-404-isvc-0f5d7-predictor.18c642b6ac2d0acd namespace: kserve-ci-e2e-test resourceVersion: "22442" uid: aedfc66f-e0fa-4702-b40a-edcb6fc0819a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22121" uid: 6e43f071-5c26-4fb5-997e-8d9a8b3778ea kind: Event lastTimestamp: "2026-07-27T21:27:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:49Z" name: error-404-isvc-0f5d7-predictor.18c642b6ac96b370 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: dab753bb-1fe1-435b-a5a3-58f119479ddc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:28:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22304" uid: 6e43f071-5c26-4fb5-997e-8d9a8b3778ea kind: Event lastTimestamp: "2026-07-27T21:28:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:34Z" name: error-404-isvc-0f5d7-predictor.18c642bda9f07f5e namespace: kserve-ci-e2e-test resourceVersion: "22884" uid: 9bd4d0ab-5855-46e1-bf60-ed29c0dab5ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:28:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22304" uid: 6e43f071-5c26-4fb5-997e-8d9a8b3778ea kind: Event lastTimestamp: "2026-07-27T21:28:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:34Z" name: error-404-isvc-0f5d7-predictor.18c642bdaa64e2ea namespace: kserve-ci-e2e-test resourceVersion: "22885" uid: 9aa68978-3344-41f5-8660-08e4c1e3f64a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: c99f7f23-bd2d-4114-8926-cac99338f195 kind: Event lastTimestamp: "2026-07-27T21:28:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:28:07Z" name: error-404-isvc-0f5d7.18c642b32ddf2543 namespace: kserve-ci-e2e-test resourceVersion: "22641" uid: f30b7d7c-15b7-4eb9-8e28-c53b8eb1602a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: c99f7f23-bd2d-4114-8926-cac99338f195 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: 'Failed to update status for InferenceService "error-404-isvc-0f5d7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0f5d7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:27:19Z" name: error-404-isvc-0f5d7.18c642b32e82c73a namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: 2ac6349a-db85-4cfd-b7bf-eaf2e18cd3b7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: c99f7f23-bd2d-4114-8926-cac99338f195 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0f5d7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:27:19Z" name: error-404-isvc-0f5d7.18c642b32e830383 namespace: kserve-ci-e2e-test resourceVersion: "22128" uid: f5856de5-50ef-45e5-8f75-ea08f2e0d72e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22639" uid: c99f7f23-bd2d-4114-8926-cac99338f195 kind: Event lastTimestamp: "2026-07-27T21:28:07Z" message: InferenceService [error-404-isvc-0f5d7] is Ready metadata: creationTimestamp: "2026-07-27T21:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:28:07Z" name: error-404-isvc-0f5d7.18c642be55d53b40 namespace: kserve-ci-e2e-test resourceVersion: "22640" uid: 2c48a071-db80-4f86-a4f5-a5653acebc81 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:36:04.774616Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26318" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-61561-predictor-545d79d4f6-h8m88 to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:36:04Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432d729f8e8a namespace: kserve-ci-e2e-test resourceVersion: "26322" uid: 1345dffb-98e8-4d5b-97a7-8ad05aea0e22 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-61561-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:04Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432d7e38015e namespace: kserve-ci-e2e-test resourceVersion: "26352" uid: 5147dcf2-f91d-4416-9f2f-3a6d279343c1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26335" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:36:05Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432dafda1f5f namespace: kserve-ci-e2e-test resourceVersion: "26370" uid: 99b8cb15-2e52-42ca-a8c9-c29856508d21 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432db12bf34d namespace: kserve-ci-e2e-test resourceVersion: "26375" uid: 1591b85b-0036-494b-84f4-cf5b6d6f8a83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432db4aa06b6 namespace: kserve-ci-e2e-test resourceVersion: "26378" uid: a3f1f93b-f0df-477f-8603-56789dfb890c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432db521dbe8 namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: 164130e7-4b90-486a-bda1-6d6875d2f4d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432db52e3b38 namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 4014743f-b0cf-4bc0-8441-1bd35866b7d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432db852da0d namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: b50bd229-8de2-4684-8558-9e88a7605547 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432db8ceb605 namespace: kserve-ci-e2e-test resourceVersion: "26382" uid: 67b40298-3020-49aa-ac84-8e07c788f80e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:36:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:36:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:42Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6432dca25870d namespace: kserve-ci-e2e-test resourceVersion: "26827" uid: 4af0a567-641a-4c1d-b52c-1318f810ac59 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6433dcdf842ed namespace: kserve-ci-e2e-test resourceVersion: "27212" uid: db1c0925-b22a-4a1d-8a62-1a9cfe5797e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6433dcdf8c2a1 namespace: kserve-ci-e2e-test resourceVersion: "27218" uid: 12dee4a9-4c9a-4c9b-b671-d01d2baaaa70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-61561-predictor-545d79d4f6-h8m88 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 8fa05a7b-4022-4733-8d01-4e8a990dd69a kind: Event lastTimestamp: "2026-07-27T21:37:17Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:17Z" name: error-404-isvc-61561-predictor-545d79d4f6-h8m88.18c6433e523ec16a namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 2e6daf26-27a4-48ba-875b-7f0b2c65fd4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-61561-predictor-545d79d4f6 namespace: kserve-ci-e2e-test resourceVersion: "26311" uid: a29a1151-93a2-4342-8258-2183c0021e70 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: 'Created pod: error-404-isvc-61561-predictor-545d79d4f6-h8m88' metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:04Z" name: error-404-isvc-61561-predictor-545d79d4f6.18c6432d72366eb4 namespace: kserve-ci-e2e-test resourceVersion: "26319" uid: 7f560757-47e7-4529-b3f9-b658f77c90ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 7ac86484-b12a-4c59-b028-a5a3ee6d29f8 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: Scaled up replica set error-404-isvc-61561-predictor-545d79d4f6 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:04Z" name: error-404-isvc-61561-predictor.18c6432d7091088f namespace: kserve-ci-e2e-test resourceVersion: "26312" uid: 835380af-4970-4a5d-b60e-e34731223008 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: 86e5e4f7-8704-4a57-99bd-8fdb34bb895e kind: Event lastTimestamp: "2026-07-27T21:36:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:19Z" name: error-404-isvc-61561-predictor.18c64330eef7fb46 namespace: kserve-ci-e2e-test resourceVersion: "26504" uid: e3c1d023-0aaf-4472-a629-103836ba8ef5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: 86e5e4f7-8704-4a57-99bd-8fdb34bb895e kind: Event lastTimestamp: "2026-07-27T21:36:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:19Z" name: error-404-isvc-61561-predictor.18c64330ef5e8d1f namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: 6be017f5-6156-49b7-83d8-68ef87a0f088 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: 86e5e4f7-8704-4a57-99bd-8fdb34bb895e kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: error-404-isvc-61561-predictor.18c643346ded4bda namespace: kserve-ci-e2e-test resourceVersion: "27052" uid: 4fff3613-b764-49bd-b81c-ee1975e981e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: 86e5e4f7-8704-4a57-99bd-8fdb34bb895e kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: error-404-isvc-61561-predictor.18c643346e402475 namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 6e5ce0a0-6baa-44d6-b266-1a40ebcaadb9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-61561 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 814ae8c1-c42c-4650-afcb-30a96b044290 kind: Event lastTimestamp: "2026-07-27T21:36:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:52Z" name: error-404-isvc-61561.18c6432d70918811 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: cc309ede-05f6-4f87-bc02-f53860d131df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-61561 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 814ae8c1-c42c-4650-afcb-30a96b044290 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-61561": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-61561": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:04Z" name: error-404-isvc-61561.18c6432d71281b34 namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: 37912e4f-165f-4f4a-9afb-57f1234754e9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-61561 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 814ae8c1-c42c-4650-afcb-30a96b044290 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-61561": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:04Z" name: error-404-isvc-61561.18c6432d7128619a namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: e72a4d0b-0359-4d6d-b01c-749ef8a387e6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-61561 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 814ae8c1-c42c-4650-afcb-30a96b044290 kind: Event lastTimestamp: "2026-07-27T21:36:52Z" message: InferenceService [error-404-isvc-61561] is Ready metadata: creationTimestamp: "2026-07-27T21:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:52Z" name: error-404-isvc-61561.18c643388523bf9e namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 70d10bab-e5f5-44d8-a010-259583b988e4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:37:15.128144Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27239" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c6433dd40532d6 namespace: kserve-ci-e2e-test resourceVersion: "27243" uid: dc3e79ad-aebf-45e1-b071-000885e2a5c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27247" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c6433dee0da7c4 namespace: kserve-ci-e2e-test resourceVersion: "27264" uid: b89dcd7f-615b-490c-aab9-c11bd8cd35b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c6433def5b6153 namespace: kserve-ci-e2e-test resourceVersion: "27266" uid: efe3d381-45f3-4302-8471-67af459fceb8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c6433df203d123 namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: ca10403a-6b71-4abf-bbb6-2ae4962a4c0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c6433df2752e26 namespace: kserve-ci-e2e-test resourceVersion: "27269" uid: ce1d5bf7-3c3e-48a0-a86f-2cf94f799d86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c6433df27f86d6 namespace: kserve-ci-e2e-test resourceVersion: "27270" uid: f091ad5c-d0e0-4232-9b0f-c6f66010c02d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c6433df566304e namespace: kserve-ci-e2e-test resourceVersion: "27271" uid: 7e2d2015-ea3b-4f5e-b5af-5f25c75291e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c6433df5f71e67 namespace: kserve-ci-e2e-test resourceVersion: "27272" uid: 5fdfa5f7-4eaf-475d-8c4f-7ad098a4449d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:37:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:37:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:53Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c6433e619e101c namespace: kserve-ci-e2e-test resourceVersion: "27658" uid: 98fd5e6a-7836-45f1-9c77-277cebec7dc6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:46:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:46:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:46:29Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c643befbf57b30 namespace: kserve-ci-e2e-test resourceVersion: "31332" uid: f5c0208d-6a04-4ee3-bb07-a13dc1c54b54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:46:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: 8df6bafa-cbe8-407a-a34c-e336c29ae291 kind: Event lastTimestamp: "2026-07-27T21:46:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:46:29Z" name: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn.18c643befbf60b32 namespace: kserve-ci-e2e-test resourceVersion: "31333" uid: 71b0451a-5a33-4b89-bea9-d7679c11121c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b2e96-predictor-7557b79bc4 namespace: kserve-ci-e2e-test resourceVersion: "27232" uid: 56e86fbf-c022-43b2-9807-5baa186ba4f4 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: 'Created pod: error-404-isvc-b2e96-predictor-7557b79bc4-bfcbn' metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor-7557b79bc4.18c6433dd3925fb1 namespace: kserve-ci-e2e-test resourceVersion: "27240" uid: 9393bc54-3b88-4f97-be83-a40ceeba8b71 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27228" uid: c8bb0d19-dc92-42e7-854f-70d0fa4f3926 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: Scaled up replica set error-404-isvc-b2e96-predictor-7557b79bc4 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96-predictor.18c6433dd1f7b4f7 namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 6c0c3a1c-9324-4bbe-80e8-6c2e472d1f7d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:37:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27230" uid: 06307fc7-f620-4fb8-8b73-d217ea7ca2ed kind: Event lastTimestamp: "2026-07-27T21:38:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:45Z" name: error-404-isvc-b2e96-predictor.18c643415070a58f namespace: kserve-ci-e2e-test resourceVersion: "28093" uid: 130934ff-68a8-4965-a88d-1b59a2e44429 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:37:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27230" uid: 06307fc7-f620-4fb8-8b73-d217ea7ca2ed kind: Event lastTimestamp: "2026-07-27T21:38:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:45Z" name: error-404-isvc-b2e96-predictor.18c6434150b888f9 namespace: kserve-ci-e2e-test resourceVersion: "28094" uid: 556860c0-6b7a-4c82-875d-96e191c50a93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27432" uid: 06307fc7-f620-4fb8-8b73-d217ea7ca2ed kind: Event lastTimestamp: "2026-07-27T21:37:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:45Z" name: error-404-isvc-b2e96-predictor.18c64344cf53e515 namespace: kserve-ci-e2e-test resourceVersion: "27591" uid: 9afdf38b-4401-409b-a310-70b0cf267d40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27432" uid: 06307fc7-f620-4fb8-8b73-d217ea7ca2ed kind: Event lastTimestamp: "2026-07-27T21:37:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:45Z" name: error-404-isvc-b2e96-predictor.18c64344cfb51990 namespace: kserve-ci-e2e-test resourceVersion: "27593" uid: de1f0fdc-fc46-41f1-a829-5d62890b77d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: a549298d-7c7d-4dd6-8e49-a72a5570fb16 kind: Event lastTimestamp: "2026-07-27T21:38:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:38:03Z" name: error-404-isvc-b2e96.18c6433dd1fbb793 namespace: kserve-ci-e2e-test resourceVersion: "27747" uid: 788a4885-beb1-4424-b7bc-eda2c8ebe3b2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: a549298d-7c7d-4dd6-8e49-a72a5570fb16 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-b2e96": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2e96": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96.18c6433dd2a33070 namespace: kserve-ci-e2e-test resourceVersion: "27236" uid: 32d4c4ea-99ce-4689-97e8-d113d78ecb2d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: a549298d-7c7d-4dd6-8e49-a72a5570fb16 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2e96": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:37:15Z" name: error-404-isvc-b2e96.18c6433dd2a38645 namespace: kserve-ci-e2e-test resourceVersion: "27237" uid: 9e0fcda3-cc0c-4f68-bffc-698ced1cae8d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:38:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27740" uid: a549298d-7c7d-4dd6-8e49-a72a5570fb16 kind: Event lastTimestamp: "2026-07-27T21:38:03Z" message: InferenceService [error-404-isvc-b2e96] is Ready metadata: creationTimestamp: "2026-07-27T21:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:38:03Z" name: error-404-isvc-b2e96.18c643491ccdf2b5 namespace: kserve-ci-e2e-test resourceVersion: "27741" uid: a8235793-72ed-475f-8a25-e0c77a15c20e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:36:34.576689Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26708" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-beb6a-predictor-7c99f78746-rr2sm to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:36:34Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c6433462f6ffee namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: a2a34b17-58b2-491a-a3cf-49ad4a4f075e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:36:35Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:36:35Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c643347d1360ca namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: a3bf88d4-6ad4-4d2c-9f96-7900579b64e7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:36:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:35Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c643347e68df1d namespace: kserve-ci-e2e-test resourceVersion: "26736" uid: cb0e5d85-ce07-4a34-9f4d-e2e32bb97783 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:36:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:35Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c6433481a7f3e1 namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: a5e227d2-caaf-4cff-8e21-871011862a52 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:36:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:35Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c64334822e6367 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: c4b4353a-acce-4683-8aa9-fcf4f7f21aa3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:36:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:35Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c643348238f382 namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: 1d2d690d-67f4-4c7c-91f3-39ebc6be755c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:36:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:35Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c64334856c2c82 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: db0f2577-53bf-4761-a7e6-434393a878d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:36:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:35Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c6433485e2b4eb namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: b305c5ee-7538-43fe-b3e2-100ec60b8992 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:37:12Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:12Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c64334cce8247a namespace: kserve-ci-e2e-test resourceVersion: "27127" uid: 106ffab6-4c65-411f-8aac-cab49677d2af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:53:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:53:53Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c6442658c3d10e namespace: kserve-ci-e2e-test resourceVersion: "34419" uid: a5326ea6-a367-4792-be6c-989c77ef35db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: 240e7788-4b88-4b27-984f-0a35b90ceaa3 kind: Event lastTimestamp: "2026-07-27T21:53:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:53:53Z" name: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm.18c6442658c468c5 namespace: kserve-ci-e2e-test resourceVersion: "34420" uid: 864a571e-89c1-49aa-833c-9c92251d1259 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-beb6a-predictor-7c99f78746 namespace: kserve-ci-e2e-test resourceVersion: "26701" uid: 4fbe480f-e040-4298-b807-2a48b7a84a2e kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: 'Created pod: error-404-isvc-beb6a-predictor-7c99f78746-rr2sm' metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:34Z" name: error-404-isvc-beb6a-predictor-7c99f78746.18c64334628a05a8 namespace: kserve-ci-e2e-test resourceVersion: "26712" uid: d3964c80-18dc-4c61-a98f-f23875bff7a9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: e027a4bc-998e-4ba9-86e4-066a67bf4733 kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: Scaled up replica set error-404-isvc-beb6a-predictor-7c99f78746 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:34Z" name: error-404-isvc-beb6a-predictor.18c6433460d1e6cc namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: b39979f5-9dd9-47a0-9157-a80a1488a717 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:36:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26700" uid: e463da21-1bee-439d-902c-6e1b723c9b0b kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:36:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: error-404-isvc-beb6a-predictor.18c64337df4e8dc2 namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: 489cba3f-eb1b-4c6c-9c5d-c9107189e546 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:36:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26700" uid: e463da21-1bee-439d-902c-6e1b723c9b0b kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:36:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: error-404-isvc-beb6a-predictor.18c64337dfbe33ff namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: e9751660-6010-4be8-9f33-16de63cad99b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:37:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: e463da21-1bee-439d-902c-6e1b723c9b0b kind: Event lastTimestamp: "2026-07-27T21:37:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:49Z" name: error-404-isvc-beb6a-predictor.18c6433edd16cd09 namespace: kserve-ci-e2e-test resourceVersion: "27630" uid: 4a9ca85e-959c-4983-a76e-416e299d9ce2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:37:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: e463da21-1bee-439d-902c-6e1b723c9b0b kind: Event lastTimestamp: "2026-07-27T21:37:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:49Z" name: error-404-isvc-beb6a-predictor.18c6433edd5c8b36 namespace: kserve-ci-e2e-test resourceVersion: "27631" uid: eb6376bc-5ce9-4c55-a3e5-44e974f49ca8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-beb6a namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: 119a1c61-ada3-4193-a4f2-98c97b5821ea kind: Event lastTimestamp: "2026-07-27T21:37:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:37:22Z" name: error-404-isvc-beb6a.18c6433460e6597e namespace: kserve-ci-e2e-test resourceVersion: "27361" uid: b92f182d-88a2-48df-8c43-ccc70edae74b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-beb6a namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: 119a1c61-ada3-4193-a4f2-98c97b5821ea kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: 'Failed to update status for InferenceService "error-404-isvc-beb6a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-beb6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:34Z" name: error-404-isvc-beb6a.18c643346184fe88 namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: b4bfe994-147d-470e-b6f9-8384332938d9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-beb6a namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: 119a1c61-ada3-4193-a4f2-98c97b5821ea kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-beb6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:34Z" name: error-404-isvc-beb6a.18c6433461853f30 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 0fff7781-a3e2-4bca-b85d-22742b25df95 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-beb6a namespace: kserve-ci-e2e-test resourceVersion: "27359" uid: 119a1c61-ada3-4193-a4f2-98c97b5821ea kind: Event lastTimestamp: "2026-07-27T21:37:22Z" message: InferenceService [error-404-isvc-beb6a] is Ready metadata: creationTimestamp: "2026-07-27T21:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:37:22Z" name: error-404-isvc-beb6a.18c6433f87e4d503 namespace: kserve-ci-e2e-test resourceVersion: "27360" uid: 05fc9330-fe5b-4d2d-91fb-1cab865fc3bf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:25:39.659564Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:25:39Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c6429be6dcd0a6 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: bc5fa0a4-5313-4130-92cc-94773bf6a170 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:25:40Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c6429c002c3b9b namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: 09e6e532-222e-4c03-a782-17b7e80ef058 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c6429c0170c3e3 namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 8f9e45df-7841-42eb-bd72-f5423a1d7a9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c6429c044d4972 namespace: kserve-ci-e2e-test resourceVersion: "20823" uid: befe1707-9a07-4d28-a800-3b8a7efaec7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c6429c04cb2556 namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: 13d2dc1d-17c4-4952-a33c-2178127d8fa6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c6429c04d5a461 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: f9cfdd40-4027-45f4-bdbf-6e340fdba6d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c6429c0839ada7 namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 9c498799-9108-4c7c-9201-d8fa9e1b3c98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c6429c08b0e4b7 namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: 93c7256c-d983-4e9e-821c-d6d79a200541 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:25:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:26:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:25:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:18Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c6429c781ed8c0 namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 1301abb0-3639-4991-b05f-8a7b05cfef9d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:49Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c642ac3b2617cb namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: 4e99f2d4-f469-4d0b-98b7-9582d305a5a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 791d971c-ea7d-4d22-930f-e6d50e2b9151 kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:49Z" name: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc.18c642ac3b26ad1e namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 392f193d-8b1e-450f-931b-86ec2614610b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cb683-predictor-84d5bf79d6 namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: dbfd1e02-6663-494a-8c61-712180b2ba9e kind: Event lastTimestamp: "2026-07-27T21:25:39Z" message: 'Created pod: error-404-isvc-cb683-predictor-84d5bf79d6-bz4dc' metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:25:39Z" name: error-404-isvc-cb683-predictor-84d5bf79d6.18c6429be66f38dd namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: f1e6422d-ddd6-4b08-9b55-53ff7122202b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 2ffeec9f-bd57-4a2b-8e4c-feb32978739d kind: Event lastTimestamp: "2026-07-27T21:25:39Z" message: Scaled up replica set error-404-isvc-cb683-predictor-84d5bf79d6 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:25:39Z" name: error-404-isvc-cb683-predictor.18c6429be4d0c1e8 namespace: kserve-ci-e2e-test resourceVersion: "20787" uid: 18ea9d55-5476-493e-89a2-c056f27b109f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 2f8db9f2-38aa-405a-abee-4b2ce355c30f kind: Event lastTimestamp: "2026-07-27T21:25:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:25:54Z" name: error-404-isvc-cb683-predictor.18c6429f6349ec13 namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: aafcae02-c1db-49a7-82e1-baddabe91fa5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 2f8db9f2-38aa-405a-abee-4b2ce355c30f kind: Event lastTimestamp: "2026-07-27T21:25:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:25:54Z" name: error-404-isvc-cb683-predictor.18c6429f63ac8e78 namespace: kserve-ci-e2e-test resourceVersion: "20953" uid: 1dc044f9-be8e-452a-97e9-f54b920fa39d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: 2f8db9f2-38aa-405a-abee-4b2ce355c30f kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: error-404-isvc-cb683-predictor.18c642a2e25635b1 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: 1eb13973-98d7-4815-8c37-be108ef7297a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: 2f8db9f2-38aa-405a-abee-4b2ce355c30f kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: error-404-isvc-cb683-predictor.18c642a2e2c367e7 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 065cf609-4f66-49fe-9c9d-2c01e0896ee6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb683 namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: a54c73bf-d10b-4bf3-b300-3b58f107d004 kind: Event lastTimestamp: "2026-07-27T21:26:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:28Z" name: error-404-isvc-cb683.18c6429be4e33d75 namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: fbf13e32-5868-4222-9a00-6978c252ddfc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb683 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: a54c73bf-d10b-4bf3-b300-3b58f107d004 kind: Event lastTimestamp: "2026-07-27T21:26:28Z" message: InferenceService [error-404-isvc-cb683] is Ready metadata: creationTimestamp: "2026-07-27T21:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:28Z" name: error-404-isvc-cb683.18c642a7339813dd namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: e18a0b59-89e1-4863-898f-fa206a561991 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:16:54.676313Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16856" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d13b2-predictor-557774b68b-xp7n6 to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:16:54Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c64221ab6c18ae namespace: kserve-ci-e2e-test resourceVersion: "16862" uid: c9093f71-8b0b-4801-ab49-8229e1dba3d5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d13b2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c64221b8d97d38 namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: 2a8c5040-f5fd-4e4d-9781-3b8ba910838b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16871" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:16:55Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c64221e86ec493 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 30a6cb7f-3003-454e-a0d6-11992977704c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c64221e9bfb311 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: ba7df605-4c5c-41f3-8231-bd76087432aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c64221ec84f25b namespace: kserve-ci-e2e-test resourceVersion: "16909" uid: 853d7785-22c8-4b5e-982f-4a3dcceca4a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c64221ecf96e6c namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: 9d12d305-90bf-47ab-885d-4ab0dece0466 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c64221ed0274f9 namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 9a71f6cd-e924-4da8-9b14-0979b0cfe403 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c64221eff7e7a6 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: e266ae5d-834a-48ad-8d01-5ae67f9d08f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c64221f06c7ebf namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: ecb6b264-41ef-4cda-9655-41dc39b9ff10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-27T21:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:26:12Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:12Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c642220dba94c0 namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: 18ecb878-b953-4c84-b260-43bc2614e5c0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:09Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c642a2d0e1cd39 namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 66144e0a-39f4-404f-b26d-1eb21733e317 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:09Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c642a2d0e27759 namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: 449e9ad5-4b0f-41de-b9bb-42b0316852b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 840489b6-c02a-4575-903d-99faaf200107 kind: Event lastTimestamp: "2026-07-27T21:26:12Z" message: 'Readiness probe failed: Get "https://10.133.0.19:8643/healthz": dial tcp 10.133.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:12Z" name: error-404-isvc-d13b2-predictor-557774b68b-xp7n6.18c642a38221ee67 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: eec72f79-1069-4533-a4cb-c0ad8bab487a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d13b2-predictor-557774b68b namespace: kserve-ci-e2e-test resourceVersion: "16849" uid: 817b6276-8b5f-45ba-baa8-0d2937ad9bd8 kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: 'Created pod: error-404-isvc-d13b2-predictor-557774b68b-xp7n6' metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:54Z" name: error-404-isvc-d13b2-predictor-557774b68b.18c64221aaf64ab0 namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: 88959cae-0935-4b94-bdc8-d7f67e6b57ca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: ff007d3e-5597-4035-9993-3a838babee59 kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Scaled up replica set error-404-isvc-d13b2-predictor-557774b68b from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:54Z" name: error-404-isvc-d13b2-predictor.18c64221a8c3e7bb namespace: kserve-ci-e2e-test resourceVersion: "16850" uid: eb97fb07-38d3-4c2d-9d4f-3826e88617eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:17:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16847" uid: 443b57e6-2e4b-4d20-874b-8ddfc757e3c6 kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:54Z" name: error-404-isvc-d13b2-predictor.18c6422527246f69 namespace: kserve-ci-e2e-test resourceVersion: "17489" uid: 6d2344e5-fe8e-4377-bb32-e1e0ddf7eb5f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:17:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16847" uid: 443b57e6-2e4b-4d20-874b-8ddfc757e3c6 kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:54Z" name: error-404-isvc-d13b2-predictor.18c64225276d3fc8 namespace: kserve-ci-e2e-test resourceVersion: "17490" uid: d60a200d-216b-4887-a03f-d1c0d586fb0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:18:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17042" uid: 443b57e6-2e4b-4d20-874b-8ddfc757e3c6 kind: Event lastTimestamp: "2026-07-27T21:18:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:18:09Z" name: error-404-isvc-d13b2-predictor.18c6423323be9b31 namespace: kserve-ci-e2e-test resourceVersion: "17619" uid: cb2a7d9a-6655-42c0-bc99-c6798dd0de7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:18:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17042" uid: 443b57e6-2e4b-4d20-874b-8ddfc757e3c6 kind: Event lastTimestamp: "2026-07-27T21:18:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:18:09Z" name: error-404-isvc-d13b2-predictor.18c642332426c660 namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: fe09b92f-18c1-49e5-8ce2-8f615fef4cab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: 8a8ba0fa-e11e-4901-8927-b267deab8263 kind: Event lastTimestamp: "2026-07-27T21:17:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:42Z" name: error-404-isvc-d13b2.18c64221a8bf205e namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: be853e85-ed88-4b46-9c13-8dab57d0fde9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "17372" uid: 8a8ba0fa-e11e-4901-8927-b267deab8263 kind: Event lastTimestamp: "2026-07-27T21:17:42Z" message: InferenceService [error-404-isvc-d13b2] is Ready metadata: creationTimestamp: "2026-07-27T21:17:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:42Z" name: error-404-isvc-d13b2.18c6422cc87f92d6 namespace: kserve-ci-e2e-test resourceVersion: "17373" uid: 7f4ffa7c-46ba-464a-bdea-0e92e6e2108d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:16:24.708634Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d355e-predictor-77499898bb-cghz4 to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:16:24Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421ab135a054 namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 8bba3f67-f549-46b0-a0d6-ce6ff0734170 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d355e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:24Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421abc10f00a namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 20c8bb06-f6bd-4db5-ac96-b1261f960612 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:16:25Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421aee4181ce namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: e44f5eac-6542-4b25-b417-2658ff24bfcc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421aef91fa0b namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: edd74dc5-daf7-46fb-b0e6-fedb2db62a50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421af28869aa namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: 608d2788-57dc-4460-80df-0f977a09b1f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421af30a7935 namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 08c2c0d1-3861-40a0-aa4c-ffd1263b883a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421af313c04b namespace: kserve-ci-e2e-test resourceVersion: "16438" uid: e8f1055a-14c3-4f7c-ae1d-d224eecaca86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421af5cbfea4 namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 9d884893-9a32-4bc8-bda0-db6790f1bdb5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421af647726c namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: 6435c5b0-f0c2-4634-94fa-a6989586ca1c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-27T21:16:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:25:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:42Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6421b0aab2d9e namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: c80daf29-95c9-4bc6-8cce-290cdca0eea5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:25:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:39Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6429be1d5b28c namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: 1246d9b6-84f2-491b-b42e-4e560dcb3134 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:25:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:39Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6429be1d66374 namespace: kserve-ci-e2e-test resourceVersion: "20773" uid: 322bdfd9-e169-4ab2-afeb-d9e141cb6d36 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d355e-predictor-77499898bb-cghz4 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 1ca8209c-921e-48e2-9de4-82a1d5d410b1 kind: Event lastTimestamp: "2026-07-27T21:25:42Z" message: 'Readiness probe failed: Get "https://10.133.0.18:8643/healthz": dial tcp 10.133.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:25:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:42Z" name: error-404-isvc-d355e-predictor-77499898bb-cghz4.18c6429c7f113e2d namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: b21834f9-f7aa-4b9c-a5d5-5f6ffc4d4b6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d355e-predictor-77499898bb namespace: kserve-ci-e2e-test resourceVersion: "16358" uid: 1e3d4386-ec74-4128-bfb6-547de41c8f09 kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'Created pod: error-404-isvc-d355e-predictor-77499898bb-cghz4' metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:24Z" name: error-404-isvc-d355e-predictor-77499898bb.18c6421ab0cad2c0 namespace: kserve-ci-e2e-test resourceVersion: "16366" uid: dfe51863-c32d-4aa2-946b-35990265ed28 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: 594db447-3253-469d-8090-77d6c26e0126 kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: Scaled up replica set error-404-isvc-d355e-predictor-77499898bb from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:24Z" name: error-404-isvc-d355e-predictor.18c6421aaf578d3e namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: 1a5c6830-1cc6-4d15-8e02-ceb6d5d34a88 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: ceec5335-96f4-4195-84b4-15962a88cdc2 kind: Event lastTimestamp: "2026-07-27T21:16:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:39Z" name: error-404-isvc-d355e-predictor.18c6421e2dca7dd0 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: ee322473-f7e5-407d-9ac0-5b54196793d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: ceec5335-96f4-4195-84b4-15962a88cdc2 kind: Event lastTimestamp: "2026-07-27T21:16:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:39Z" name: error-404-isvc-d355e-predictor.18c6421e2e2c7e74 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: b1dcdd94-0f04-43d6-bba9-f28f4ec5d24d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: ceec5335-96f4-4195-84b4-15962a88cdc2 kind: Event lastTimestamp: "2026-07-27T21:17:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:39Z" name: error-404-isvc-d355e-predictor.18c64221ace82962 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 3b8734cd-cc37-4114-8094-b71d68253b5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: ceec5335-96f4-4195-84b4-15962a88cdc2 kind: Event lastTimestamp: "2026-07-27T21:17:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:39Z" name: error-404-isvc-d355e-predictor.18c64221ad6f7509 namespace: kserve-ci-e2e-test resourceVersion: "17337" uid: a25cb7ff-f9c7-484d-8216-0e357e179c94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d355e namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: 7cc0d690-c9bc-48ac-a8e2-f7b62a2f038e kind: Event lastTimestamp: "2026-07-27T21:17:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:12Z" name: error-404-isvc-d355e.18c6421aaf621ade namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: 8d5830a5-90c1-493a-8d5c-42ff854231ea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d355e namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: 7cc0d690-c9bc-48ac-a8e2-f7b62a2f038e kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'Failed to update status for InferenceService "error-404-isvc-d355e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d355e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:24Z" name: error-404-isvc-d355e.18c6421aafd70cc9 namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 87014afc-58bb-4a6a-9319-86279b6508ef reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d355e namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: 7cc0d690-c9bc-48ac-a8e2-f7b62a2f038e kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d355e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:24Z" name: error-404-isvc-d355e.18c6421aafd74e0e namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 7cb03a7b-0f73-4413-88cb-bddf5edae6b2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d355e namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7cc0d690-c9bc-48ac-a8e2-f7b62a2f038e kind: Event lastTimestamp: "2026-07-27T21:17:12Z" message: InferenceService [error-404-isvc-d355e] is Ready metadata: creationTimestamp: "2026-07-27T21:17:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:12Z" name: error-404-isvc-d355e.18c64225c57ddbcd namespace: kserve-ci-e2e-test resourceVersion: "17070" uid: 96d7a482-4ef5-4e43-ba88-f0616f140501 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:26:49.987181Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:26:49Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac46b712a7 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: e11c0459-bf5d-4c86-bc3e-9f34ff2bebcc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:26:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e26c5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:50Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac51d39569 namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: 47628f23-5118-4df3-a822-282784908d4f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:26:51Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:26:51Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac83c16bc2 namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: 119e90bc-8844-4319-b38e-4aa8cc649dff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:26:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:51Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac8520f283 namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 9f005f15-2a08-47f7-b26d-d23c91faa883 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:26:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:51Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac882a3b07 namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: cc25dba7-802c-4db5-9b23-5ecf673017cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:26:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:51Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac88e17b1e namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 658ea149-2ba2-4eb3-a6f1-ed0cf2f00b1c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:26:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:51Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac88ee2ac5 namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: d1f81176-92d8-4a64-8e40-d1c4d1d7c0c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:26:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:51Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac8ba39f86 namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: aa295340-bd62-4922-bd5b-fbaeba2575e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:26:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:51Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac8c176d8c namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 155edb60-b1d0-40b0-b1ee-ab597ded0a57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-27T21:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:36:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:07Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c642ac97e3efac namespace: kserve-ci-e2e-test resourceVersion: "26404" uid: 16bd75f9-d45c-481c-b282-5ea7053c83ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:04Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c6432d75ddae32 namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: 2214ecf9-b1b9-4a1e-833b-14549c2fd2d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:04Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c6432d75de71aa namespace: kserve-ci-e2e-test resourceVersion: "26344" uid: e8cc17e3-6bd7-48e7-b181-02b314a62221 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: e12f0e51-4d30-47ec-9730-7f1f552d3461 kind: Event lastTimestamp: "2026-07-27T21:36:07Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:07Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7.18c6432e0c40965c namespace: kserve-ci-e2e-test resourceVersion: "26403" uid: 822548f1-d9c6-4ff3-86e1-0732e2ca2953 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e26c5-predictor-7897bcc8dd namespace: kserve-ci-e2e-test resourceVersion: "21688" uid: 712fd924-1f41-4b80-b99e-eed2d487df05 kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: 'Created pod: error-404-isvc-e26c5-predictor-7897bcc8dd-zplm7' metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:49Z" name: error-404-isvc-e26c5-predictor-7897bcc8dd.18c642ac464fce9b namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: 6d478d18-df6e-4a60-a4a8-15a728772f35 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: efa9eb6c-18d5-4880-8f59-d138bfd68461 kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: Scaled up replica set error-404-isvc-e26c5-predictor-7897bcc8dd from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:49Z" name: error-404-isvc-e26c5-predictor.18c642ac441bc1c5 namespace: kserve-ci-e2e-test resourceVersion: "21690" uid: 97cd8215-37ff-447a-a2b6-ac8565606f12 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:27:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 20b3dca7-a8e4-4772-8564-345888fe2c81 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:19Z" name: error-404-isvc-e26c5-predictor.18c642afc2603ae3 namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: ced3da1a-a3e2-425a-8326-44752292b462 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:27:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 20b3dca7-a8e4-4772-8564-345888fe2c81 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:19Z" name: error-404-isvc-e26c5-predictor.18c642afc2bd0197 namespace: kserve-ci-e2e-test resourceVersion: "22156" uid: 2b61cde4-8ac2-43a6-b4a1-977ae85fb90e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: 20b3dca7-a8e4-4772-8564-345888fe2c81 kind: Event lastTimestamp: "2026-07-27T21:28:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:05Z" name: error-404-isvc-e26c5-predictor.18c642b6c01104e5 namespace: kserve-ci-e2e-test resourceVersion: "22606" uid: 5c88bf46-ba0c-418d-b15c-39c0a737c307 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: 20b3dca7-a8e4-4772-8564-345888fe2c81 kind: Event lastTimestamp: "2026-07-27T21:28:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:05Z" name: error-404-isvc-e26c5-predictor.18c642b6c0759ab6 namespace: kserve-ci-e2e-test resourceVersion: "22607" uid: d6b01093-009f-4afc-8de3-d38294ca4a64 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: 725706bf-5603-4fc4-b434-f11e2580e5be kind: Event lastTimestamp: "2026-07-27T21:27:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:27:37Z" name: error-404-isvc-e26c5.18c642ac43f45cfb namespace: kserve-ci-e2e-test resourceVersion: "22349" uid: 12e91444-7fb1-452a-b9cb-37a18f3c3e73 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: 725706bf-5603-4fc4-b434-f11e2580e5be kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: 'Failed to update status for InferenceService "error-404-isvc-e26c5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e26c5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:49Z" name: error-404-isvc-e26c5.18c642ac4461833e namespace: kserve-ci-e2e-test resourceVersion: "21692" uid: 0d5c2880-d050-4281-a909-7e33676a3a27 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: 725706bf-5603-4fc4-b434-f11e2580e5be kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e26c5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:49Z" name: error-404-isvc-e26c5.18c642ac4461d167 namespace: kserve-ci-e2e-test resourceVersion: "21693" uid: 53802deb-1fd8-41e7-b90d-6c4aa2e430c4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 725706bf-5603-4fc4-b434-f11e2580e5be kind: Event lastTimestamp: "2026-07-27T21:27:37Z" message: InferenceService [error-404-isvc-e26c5] is Ready metadata: creationTimestamp: "2026-07-27T21:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:27:37Z" name: error-404-isvc-e26c5.18c642b752d634e5 namespace: kserve-ci-e2e-test resourceVersion: "22347" uid: 7c5a2ede-1c61-428e-871d-dde5373d6245 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:14:34.332005Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:14:34Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64200fe3fdf80 namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: e49a0541-d1a6-4d92-a700-a4f4db8fc24a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:14:34Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64201185b1259 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: 03395318-ef11-4468-bc52-91c28176375a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d" already present on machine metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:34Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c642011d006418 namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 280f7cd2-460c-4f0a-8be2-dfad2a9efa58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:34Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c6420120932521 namespace: kserve-ci-e2e-test resourceVersion: "15206" uid: b79d4094-59ac-46b5-a7b1-80c3f3e3d116 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:34Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64201210a987e namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 86619c27-fb73-46c9-9e46-aba167bec949 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:39Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-27T21:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:39Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c642023a25d481 namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 682753d2-c7f0-43ff-97ad-5ca9f6f0b34c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:53Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 13.905s (13.905s including waiting). Image size: 1555729331 bytes.' metadata: creationTimestamp: "2026-07-27T21:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:53Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c6420576fc42ec namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: ac860988-770d-4b02-92d8-b6c7ce3603e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:54Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64205a70646c9 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 29a927de-39e8-441b-b57c-2cf261b45580 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:54Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64205a76ac0a9 namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 9c70eda2-ce28-4a58-8b26-dff75054aa45 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-27T21:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:54Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64205a7769af5 namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 390d054e-e015-4f31-ac9d-7792cb6ff11e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:57Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.348s (3.348s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:57Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c642066f08c4e1 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 99d38163-6104-414a-a9f5-18567e7b321d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:57Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64206735223d6 namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 2c68363a-79fb-4938-a576-fd2023b8ac0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:14:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:57Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64206740b95c3 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: 48fe4349-de8d-41a8-8a4b-87a6314fd4c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:16:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:03Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64206748eb8a9 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 9803b33d-5055-4ba3-8d45-dd71e178bc5a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64221a1c3a3e4 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 7048c881-377e-476c-be52-8f9423bb0ab8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64221a1c40c0f namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: 19523612-f010-48f3-ad5b-67ce97beb429 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 0084728c-27c2-4278-9c64-5430287c1def kind: Event lastTimestamp: "2026-07-27T21:16:58Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:58Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224.18c64222a176a8ee namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: 18bbd7fa-8171-4bbd-8b47-ffaf6a04b67c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6d55f5d589 namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: a969c017-572e-450a-a145-825bc4dd677d kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6d55f5d589-gv224' metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:34Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589.18c64200fdb52909 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 3bca2149-6761-432e-b3b4-cc01174f846e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: ceeaff24-48c4-4b58-92ec-23f739f4071b kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6d55f5d589 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:34Z" name: isvc-sklearn-graph-1-predictor.18c64200f940f7e8 namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: d56ead8e-9f89-4edb-8b63-bfc65076e46f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 1e6446bd-82ca-4886-bdb7-7bef622d5681 kind: Event lastTimestamp: "2026-07-27T21:15:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:34Z" name: isvc-sklearn-graph-1-predictor.18c642047eb3722b namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 3dc3c670-7144-453c-90b7-9709279aaef6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 1e6446bd-82ca-4886-bdb7-7bef622d5681 kind: Event lastTimestamp: "2026-07-27T21:15:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:34Z" name: isvc-sklearn-graph-1-predictor.18c642047f1b3d74 namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: fa966f71-7b7c-41ec-bf47-2525a6baf4a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 1e6446bd-82ca-4886-bdb7-7bef622d5681 kind: Event lastTimestamp: "2026-07-27T21:16:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:49Z" name: isvc-sklearn-graph-1-predictor.18c6420b7d1dbedb namespace: kserve-ci-e2e-test resourceVersion: "16693" uid: a7fa8ecd-094b-4d57-b64d-4903f3c461de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 1e6446bd-82ca-4886-bdb7-7bef622d5681 kind: Event lastTimestamp: "2026-07-27T21:16:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:49Z" name: isvc-sklearn-graph-1-predictor.18c6420b7d7fe330 namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: 769a2908-a971-424b-a279-172bac23508b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: fe1877f9-c8b4-463a-a280-e3f2102757f7 kind: Event lastTimestamp: "2026-07-27T21:16:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:13Z" name: isvc-sklearn-graph-1.18c64200ff9bd0c1 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 21a11258-656a-4a19-894b-2646130d1b07 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: fe1877f9-c8b4-463a-a280-e3f2102757f7 kind: Event lastTimestamp: "2026-07-27T21:16:13Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-27T21:16:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:13Z" name: isvc-sklearn-graph-1.18c642182daaa20e namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: 7d3e76ac-e1bf-45f3-8e3f-551566f4b7c1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:14:35.009838Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:14:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:14:35Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c6420126a6cd99 namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 03378de4-361c-4523-9952-8a495637035f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:35Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c642013393aee7 namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: b5cd457a-004d-4e3d-9331-e698d7289706 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:36Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:14:36Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c64201649c480f namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 6c9f9098-a4c6-41e6-a636-0448fc1afd8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d" already present on machine metadata: creationTimestamp: "2026-07-27T21:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:36Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c6420166115d7d namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 89f52c68-9243-4a6b-aa21-52e9c6ccb596 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-27T21:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:36Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c642016927ab46 namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 6e2d4dab-374c-4469-9073-05e80f2db681 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-27T21:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:36Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c64201699d68e1 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: 5696229f-7a49-462b-ac0d-7a4beb90754e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:41Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-27T21:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:41Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c64202b27bfdce namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: cc94ba09-d6af-4681-be22-b838b52d88c7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:53Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 11.886s (11.886s including waiting). Image size: 1555729331 bytes.' metadata: creationTimestamp: "2026-07-27T21:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:53Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c6420576fce091 namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 8bd2b523-63cb-4bfc-944a-4e8f73827b01 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:54Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c64205a7012bc4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: a50f7e03-eef9-4c27-877b-ba2ebbf4ef96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:54Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c64205a763f82d namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: 7cf59e5e-4a3a-4dcc-9dbe-a7645e162955 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-27T21:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:54Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c64205a77250b6 namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 7a0b4083-aab0-42b3-849f-1e0ee56e060c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:57Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.326s (3.326s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:57Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c642066db7cdb4 namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 99222c26-f203-4800-9a02-bfe39b6236f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:57Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c64206735972aa namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: d0c1176a-dd1c-402f-a196-80a40fb51fa9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:14:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:57Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c642067450c938 namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: 80b637f6-f1f6-435f-814c-76bce3c5c4ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c6420674cbf356 namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: 1a112e9b-05bc-43a9-bfa5-638aa3878608 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c642219e68feb6 namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: 00df97aa-5fec-4d3f-9648-9432b5537584 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c642219e696ca7 namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: 11e1d4d0-4747-460a-ae09-2b5262da317b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 4400b7e1-c34f-47e2-b9cf-a55f0f73e9fc kind: Event lastTimestamp: "2026-07-27T21:16:59Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:59Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2.18c64221b379c4d2 namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: fd47c185-c24e-4cc5-b546-ff246f867fd0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7d44765f95 namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: 4ddab252-5a4c-494c-9b3d-d07dfc818fae kind: Event lastTimestamp: "2026-07-27T21:14:35Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7d44765f95-bggx2' metadata: creationTimestamp: "2026-07-27T21:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:35Z" name: isvc-sklearn-graph-2-predictor-7d44765f95.18c6420126232c4e namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 6764c4a6-89e7-4fb8-82f7-cb7518258169 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: 127976fb-454d-49b4-a8a5-a94fc2362788 kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7d44765f95 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:34Z" name: isvc-sklearn-graph-2-predictor.18c6420120316bee namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 5b209320-8863-4715-851a-8679f2400b37 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 955fc54c-5dad-4e4c-afdb-917bfa0ab195 kind: Event lastTimestamp: "2026-07-27T21:15:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:04Z" name: isvc-sklearn-graph-2-predictor.18c642049e5cbf4e namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: 161d627b-b524-4efe-b23b-64019d1d6136 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 955fc54c-5dad-4e4c-afdb-917bfa0ab195 kind: Event lastTimestamp: "2026-07-27T21:15:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:04Z" name: isvc-sklearn-graph-2-predictor.18c642049ed6190b namespace: kserve-ci-e2e-test resourceVersion: "15582" uid: 769abdaa-2652-4dc0-ad56-8bd31b240915 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-27T21:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 955fc54c-5dad-4e4c-afdb-917bfa0ab195 kind: Event lastTimestamp: "2026-07-27T21:16:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:50Z" name: isvc-sklearn-graph-2-predictor.18c6420b9c36658b namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: 4305f166-b0c3-46ca-b783-259a1da9ba2e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-27T21:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 955fc54c-5dad-4e4c-afdb-917bfa0ab195 kind: Event lastTimestamp: "2026-07-27T21:16:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:50Z" name: isvc-sklearn-graph-2-predictor.18c6420b9c9d5798 namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: 4777b960-b022-4e1e-9cc0-da45675a08b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: 76523db2-8f2a-4b5e-8b79-c46d28524ff3 kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:24Z" name: isvc-sklearn-graph-2.18c642011fe31df2 namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: 78ff6987-8de0-4503-9649-a12aaeb2b4b9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: 76523db2-8f2a-4b5e-8b79-c46d28524ff3 kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:24Z" name: isvc-sklearn-graph-2.18c6421abf3d8a06 namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: e9b6eea6-9e8b-4bcd-8316-9f210c3e3c1a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:14:34.626455Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-km4vt to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:14:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c642010fcccf2c namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: c598ddaf-b069-4490-997b-46867b01e457 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c6420121755525 namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 97e16e30-cae6-480c-90d1-584e74fa9b26 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:14:35Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:14:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c642014e0378b3 namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 88e9d1f2-eb76-4739-abe4-dfb18b6d8280 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:14:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d" already present on machine metadata: creationTimestamp: "2026-07-27T21:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c642014f716f43 namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: 7399af7e-073f-43dd-9a4f-9192177d5b35 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:14:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-27T21:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c6420153af1879 namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 04ca26b2-b412-403a-a78f-7906e74b4f03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:14:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-27T21:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c642015421999d namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: bd087a54-8182-404c-bb57-5bfa0bd06f90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:14:41Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-27T21:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c64202b24854c9 namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: f075647f-4047-40a3-b62d-87ed292a3840 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:15:14Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 32.935s (32.935s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-27T21:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:15:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c6420a5d615016 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 2fb47ead-e5a6-4b39-9211-1d3723e66882 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:15:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:15:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c6420a62c0ca93 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: ea794e0e-7b78-47fb-b474-d11d9bed02ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:15:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:15:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c6420a6377db34 namespace: kserve-ci-e2e-test resourceVersion: "15655" uid: 7b5f3099-5902-4656-b278-f3c14b971d48 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:15:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:15:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c6420a6395a8b7 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 1c8b4c88-1035-4a8f-a030-8a342795a31d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:15:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:15:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c6420a68e9d2a2 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: dba331c7-4321-4a36-a444-612c36940828 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:15:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:15:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c6420a6a319c03 namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: debb609c-196f-404d-b776-b9fd6b60ccec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-27T21:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:16:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c6420a7053f811 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 2a51e3a4-dbd6-4ab6-9429-108c205ca29d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c64221a77c6ed0 namespace: kserve-ci-e2e-test resourceVersion: "16845" uid: 74e982eb-da12-407d-b10a-ca960377b968 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c64221a77cf316 namespace: kserve-ci-e2e-test resourceVersion: "16848" uid: 0b65f82a-c345-422a-8751-86be5fd33675 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-km4vt namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fc01c2b2-0d3d-40dd-9f49-1ea024627a25 kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: isvc-xgboost-graph-predictor-555fc69d56-km4vt.18c64221f4965ed3 namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: 4cde0d5c-ed3c-4cf5-bf73-39196f30ef4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 4a2c9b94-9ccb-43fc-969b-1e206f9a064f kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-km4vt' metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:34Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c642010f45cbb9 namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 1a89cc35-cbee-4f72-8cdb-91fb23b88bc0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 3f8cbe36-605d-4dd5-931a-79835e8222c0 kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:34Z" name: isvc-xgboost-graph-predictor.18c642010bd6a957 namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: d821ae39-d986-40bc-8514-28ddff0094c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: f22a834c-a684-45dc-b114-d31193864967 kind: Event lastTimestamp: "2026-07-27T21:15:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:49Z" name: isvc-xgboost-graph-predictor.18c642048a4dd50b namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 2dd2204d-0722-479f-bef3-8767bb482992 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: f22a834c-a684-45dc-b114-d31193864967 kind: Event lastTimestamp: "2026-07-27T21:15:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:49Z" name: isvc-xgboost-graph-predictor.18c642048acbc437 namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 7e1794b8-8911-424b-b783-1344deaa2f5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:16:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: f22a834c-a684-45dc-b114-d31193864967 kind: Event lastTimestamp: "2026-07-27T21:16:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:49Z" name: isvc-xgboost-graph-predictor.18c64216047d34d3 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 0f055a84-2540-49b7-b8e3-493d50d2a805 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:16:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: f22a834c-a684-45dc-b114-d31193864967 kind: Event lastTimestamp: "2026-07-27T21:16:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:49Z" name: isvc-xgboost-graph-predictor.18c6421604c04a17 namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 5d2d028d-874b-46c7-90c1-777fa745140e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: bc042d4f-c206-43ec-afa7-c64042c723eb kind: Event lastTimestamp: "2026-07-27T21:16:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:31Z" name: isvc-xgboost-graph.18c642010bd324b8 namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 9eee4359-6290-4871-99e9-ebf43793c0b8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: bc042d4f-c206-43ec-afa7-c64042c723eb kind: Event lastTimestamp: "2026-07-27T21:16:30Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-27T21:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:30Z" name: isvc-xgboost-graph.18c6421c27d7ebd4 namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: d55d1dba-1f3e-4e93-b2db-990cb56e6b9a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:16:44.231900Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b84f4c8c7-h2txg namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 7aa50efa-14f9-4085-8781-43dd0901fc6a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-b84f4c8c7-h2txg to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-07-27T21:16:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:16:44Z" name: model-chainer-b84f4c8c7-h2txg.18c6421f3ce2ebb5 namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: 43411dea-cea7-4b76-92d0-d6a418fdcb65 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:44Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b84f4c8c7-h2txg namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 7aa50efa-14f9-4085-8781-43dd0901fc6a kind: Event lastTimestamp: "2026-07-27T21:16:44Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:16:44Z" name: model-chainer-b84f4c8c7-h2txg.18c6421f56e94cb9 namespace: kserve-ci-e2e-test resourceVersion: "16649" uid: 5a765c68-9c7b-4de0-9522-b0f25b4923f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-h2txg namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 7aa50efa-14f9-4085-8781-43dd0901fc6a kind: Event lastTimestamp: "2026-07-27T21:16:44Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" metadata: creationTimestamp: "2026-07-27T21:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:44Z" name: model-chainer-b84f4c8c7-h2txg.18c6421f58600c99 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: af434077-94c1-43f1-8a24-24779984221d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-h2txg namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 7aa50efa-14f9-4085-8781-43dd0901fc6a kind: Event lastTimestamp: "2026-07-27T21:16:46Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" in 1.916s (1.916s including waiting). Image size: 218232445 bytes.' metadata: creationTimestamp: "2026-07-27T21:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:46Z" name: model-chainer-b84f4c8c7-h2txg.18c6421fca96a26b namespace: kserve-ci-e2e-test resourceVersion: "16664" uid: 94593973-6b8b-4ff8-b36c-8cf454bd30df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-h2txg namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 7aa50efa-14f9-4085-8781-43dd0901fc6a kind: Event lastTimestamp: "2026-07-27T21:16:46Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-27T21:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:46Z" name: model-chainer-b84f4c8c7-h2txg.18c6421fcdcb5e4e namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: 7f6b1ba7-72b5-4276-b40c-f90cf9150e4f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-h2txg namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 7aa50efa-14f9-4085-8781-43dd0901fc6a kind: Event lastTimestamp: "2026-07-27T21:16:46Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-27T21:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:46Z" name: model-chainer-b84f4c8c7-h2txg.18c6421fce45505e namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: edf359be-7a08-4ed7-a6ab-371adbd62ea8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-h2txg namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 7aa50efa-14f9-4085-8781-43dd0901fc6a kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: model-chainer-b84f4c8c7-h2txg.18c64221977db6be namespace: kserve-ci-e2e-test resourceVersion: "16757" uid: ff32a2ef-cec7-4122-8180-54e6c047fd1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-h2txg namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 7aa50efa-14f9-4085-8781-43dd0901fc6a kind: Event lastTimestamp: "2026-07-27T21:17:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:22Z" name: model-chainer-b84f4c8c7-h2txg.18c642226c60ca6e namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: c7defe42-c813-432e-8ef7-1fd3f0d0379c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-b84f4c8c7 namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: b6656194-0de5-4933-ba63-e84c7c94122e kind: Event lastTimestamp: "2026-07-27T21:16:44Z" message: 'Created pod: model-chainer-b84f4c8c7-h2txg' metadata: creationTimestamp: "2026-07-27T21:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:44Z" name: model-chainer-b84f4c8c7.18c6421f3c8b343a namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: 8734b791-9d2f-4cee-9d18-df77b81b60e8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 03fb8966-13e0-4d3e-9d8b-04b2df463cde kind: Event lastTimestamp: "2026-07-27T21:16:44Z" message: Scaled up replica set model-chainer-b84f4c8c7 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:44Z" name: model-chainer.18c6421f3bd22d42 namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: c9dcf30c-28c2-4bcd-b06b-289189d9fca1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: ca0fddfc-5389-4822-bbf3-30697265cfa8 kind: Event lastTimestamp: "2026-07-27T21:16:44Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-27T21:16:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:44Z" name: model-chainer.18c6421f4103669b namespace: kserve-ci-e2e-test resourceVersion: "16646" uid: 0d6c755a-218e-4906-987e-8baf2065f501 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:27:09.313208Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0a095-6b6b78bdf9-crcwm namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: aa6fb367-4009-42ea-b1b7-0d36f1f2aa0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0a095-6b6b78bdf9-crcwm to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-07-27T21:27:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:27:09Z" name: sequence-graph-0a095-6b6b78bdf9-crcwm.18c642b0c6a2c381 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: 3695eff4-e5e0-41fb-be40-cfb4679093c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:09Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0a095-6b6b78bdf9-crcwm namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: aa6fb367-4009-42ea-b1b7-0d36f1f2aa0a kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:27:09Z" name: sequence-graph-0a095-6b6b78bdf9-crcwm.18c642b0e070ffda namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: 7caaf4b3-3d98-46c3-852c-dc56108a73b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0a095} kind: Pod name: sequence-graph-0a095-6b6b78bdf9-crcwm namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: aa6fb367-4009-42ea-b1b7-0d36f1f2aa0a kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-27T21:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:09Z" name: sequence-graph-0a095-6b6b78bdf9-crcwm.18c642b0e1ca90df namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 611d80fc-c801-4d53-bda6-f2e3e3bcdefd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0a095} kind: Pod name: sequence-graph-0a095-6b6b78bdf9-crcwm namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: aa6fb367-4009-42ea-b1b7-0d36f1f2aa0a kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: 'Created container: sequence-graph-0a095' metadata: creationTimestamp: "2026-07-27T21:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:09Z" name: sequence-graph-0a095-6b6b78bdf9-crcwm.18c642b0e4d08e42 namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: fbea9171-fd27-4666-baeb-d5e1452cea7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0a095} kind: Pod name: sequence-graph-0a095-6b6b78bdf9-crcwm namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: aa6fb367-4009-42ea-b1b7-0d36f1f2aa0a kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: Started container sequence-graph-0a095 metadata: creationTimestamp: "2026-07-27T21:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:09Z" name: sequence-graph-0a095-6b6b78bdf9-crcwm.18c642b0e5455f97 namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: 0d798f27-ca45-4884-9e30-51bcde5942a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0a095} kind: Pod name: sequence-graph-0a095-6b6b78bdf9-crcwm namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: aa6fb367-4009-42ea-b1b7-0d36f1f2aa0a kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: Stopping container sequence-graph-0a095 metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:19Z" name: sequence-graph-0a095-6b6b78bdf9-crcwm.18c642b31fbe4a9c namespace: kserve-ci-e2e-test resourceVersion: "22051" uid: af1032eb-8916-4666-b9b5-4ff7ec219754 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0a095} kind: Pod name: sequence-graph-0a095-6b6b78bdf9-crcwm namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: aa6fb367-4009-42ea-b1b7-0d36f1f2aa0a kind: Event lastTimestamp: "2026-07-27T21:27:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:46Z" name: sequence-graph-0a095-6b6b78bdf9-crcwm.18c642b3ad2afef3 namespace: kserve-ci-e2e-test resourceVersion: "22418" uid: dc235ee1-b3f0-456f-94d3-2ba368042a38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0a095-6b6b78bdf9 namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 404d5d55-7391-4745-8fda-a72210ca5cd2 kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: 'Created pod: sequence-graph-0a095-6b6b78bdf9-crcwm' metadata: creationTimestamp: "2026-07-27T21:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:09Z" name: sequence-graph-0a095-6b6b78bdf9.18c642b0c638ec25 namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 7a8ab469-4996-4d37-964d-278b4041743c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: 4360a3c1-92ee-4966-a033-de5b1e64060b kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: Scaled up replica set sequence-graph-0a095-6b6b78bdf9 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:09Z" name: sequence-graph-0a095.18c642b0c596f4fd namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: b029814e-d19b-438e-a401-594c91db1b92 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: d4d07ab6-1163-4fee-932d-b44eed68e119 kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: InferenceGraph [sequence-graph-0a095] is Ready metadata: creationTimestamp: "2026-07-27T21:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:27:09Z" name: sequence-graph-0a095.18c642b0ca6fd751 namespace: kserve-ci-e2e-test resourceVersion: "21943" uid: 0d85a3f2-90be-431e-a127-a90be01ee780 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:28:19.628734Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0f5d7-785897b877-f6lpp namespace: kserve-ci-e2e-test resourceVersion: "22736" uid: ff147a09-6b60-4c76-b320-782ad945d971 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0f5d7-785897b877-f6lpp to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-07-27T21:28:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:28:19Z" name: sequence-graph-0f5d7-785897b877-f6lpp.18c642c125c49653 namespace: kserve-ci-e2e-test resourceVersion: "22741" uid: f217ba83-a32f-4a26-a784-db3c5f20fb9a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:19Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0f5d7-785897b877-f6lpp namespace: kserve-ci-e2e-test resourceVersion: "22737" uid: ff147a09-6b60-4c76-b320-782ad945d971 kind: Event lastTimestamp: "2026-07-27T21:28:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-0f5d7-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:28:19Z" name: sequence-graph-0f5d7-785897b877-f6lpp.18c642c1331ba869 namespace: kserve-ci-e2e-test resourceVersion: "22756" uid: 6b0cb779-7460-4b7a-9554-a1cd1e5b2fde reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:20Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0f5d7-785897b877-f6lpp namespace: kserve-ci-e2e-test resourceVersion: "22745" uid: ff147a09-6b60-4c76-b320-782ad945d971 kind: Event lastTimestamp: "2026-07-27T21:28:20Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:28:20Z" name: sequence-graph-0f5d7-785897b877-f6lpp.18c642c1638d7dde namespace: kserve-ci-e2e-test resourceVersion: "22770" uid: ad7c3297-192a-45ee-8fbc-00d662d50191 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d7} kind: Pod name: sequence-graph-0f5d7-785897b877-f6lpp namespace: kserve-ci-e2e-test resourceVersion: "22737" uid: ff147a09-6b60-4c76-b320-782ad945d971 kind: Event lastTimestamp: "2026-07-27T21:28:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-27T21:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:28:20Z" name: sequence-graph-0f5d7-785897b877-f6lpp.18c642c164e4b8de namespace: kserve-ci-e2e-test resourceVersion: "22772" uid: 713aa76f-eed1-47f8-8bbd-13de86704a79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d7} kind: Pod name: sequence-graph-0f5d7-785897b877-f6lpp namespace: kserve-ci-e2e-test resourceVersion: "22737" uid: ff147a09-6b60-4c76-b320-782ad945d971 kind: Event lastTimestamp: "2026-07-27T21:28:20Z" message: 'Created container: sequence-graph-0f5d7' metadata: creationTimestamp: "2026-07-27T21:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:28:20Z" name: sequence-graph-0f5d7-785897b877-f6lpp.18c642c167d5cfbf namespace: kserve-ci-e2e-test resourceVersion: "22773" uid: aa660866-d6e7-437a-b89f-42e1f0b6843c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d7} kind: Pod name: sequence-graph-0f5d7-785897b877-f6lpp namespace: kserve-ci-e2e-test resourceVersion: "22737" uid: ff147a09-6b60-4c76-b320-782ad945d971 kind: Event lastTimestamp: "2026-07-27T21:28:20Z" message: Started container sequence-graph-0f5d7 metadata: creationTimestamp: "2026-07-27T21:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:28:20Z" name: sequence-graph-0f5d7-785897b877-f6lpp.18c642c1684dc761 namespace: kserve-ci-e2e-test resourceVersion: "22774" uid: 19798a84-a5bd-43ac-9b6a-bc10e08ed34e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d7} kind: Pod name: sequence-graph-0f5d7-785897b877-f6lpp namespace: kserve-ci-e2e-test resourceVersion: "22737" uid: ff147a09-6b60-4c76-b320-782ad945d971 kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: Stopping container sequence-graph-0f5d7 metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:34Z" name: sequence-graph-0f5d7-785897b877-f6lpp.18c643345184d4ad namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: bfab41ee-6e05-4474-8582-217d1b115846 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0f5d7} kind: Pod name: sequence-graph-0f5d7-785897b877-f6lpp namespace: kserve-ci-e2e-test resourceVersion: "22737" uid: ff147a09-6b60-4c76-b320-782ad945d971 kind: Event lastTimestamp: "2026-07-27T21:37:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:02Z" name: sequence-graph-0f5d7-785897b877-f6lpp.18c64334f905ae43 namespace: kserve-ci-e2e-test resourceVersion: "27005" uid: 70799409-7455-4e40-bdf0-042de06fda64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0f5d7-785897b877 namespace: kserve-ci-e2e-test resourceVersion: "22731" uid: 1cb41306-248e-4acd-a64b-88e5b94635dd kind: Event lastTimestamp: "2026-07-27T21:28:19Z" message: 'Created pod: sequence-graph-0f5d7-785897b877-f6lpp' metadata: creationTimestamp: "2026-07-27T21:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:19Z" name: sequence-graph-0f5d7-785897b877.18c642c12556b4c2 namespace: kserve-ci-e2e-test resourceVersion: "22738" uid: da3f55d1-af60-4493-93ad-2a752027e24a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22729" uid: 5a46dcbe-d423-4d86-a513-405b64a2fe04 kind: Event lastTimestamp: "2026-07-27T21:28:19Z" message: Scaled up replica set sequence-graph-0f5d7-785897b877 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:19Z" name: sequence-graph-0f5d7.18c642c1248f7778 namespace: kserve-ci-e2e-test resourceVersion: "22734" uid: b502f54f-9495-4c51-853e-dc4e217bdb2c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22748" uid: d5b24ced-ce05-466d-8cc2-b742736628b0 kind: Event lastTimestamp: "2026-07-27T21:28:19Z" message: InferenceGraph [sequence-graph-0f5d7] is Ready metadata: creationTimestamp: "2026-07-27T21:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:28:19Z" name: sequence-graph-0f5d7.18c642c1294d72bd namespace: kserve-ci-e2e-test resourceVersion: "22750" uid: 3d5ae988-767e-4d6c-910b-ac61cc6bd9c2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:28:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22732" uid: 14f7a6b7-4320-49b9-9b83-55e8ffd3c73d kind: Event lastTimestamp: "2026-07-27T21:28:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:49Z" name: sequence-graph-0f5d7.18c642c4a3cf87f9 namespace: kserve-ci-e2e-test resourceVersion: "22987" uid: 4daa3a3f-d330-48a0-983c-301324fbe6a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:28:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22732" uid: 14f7a6b7-4320-49b9-9b83-55e8ffd3c73d kind: Event lastTimestamp: "2026-07-27T21:28:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:49Z" name: sequence-graph-0f5d7.18c642c4a4137326 namespace: kserve-ci-e2e-test resourceVersion: "22988" uid: c36f6c2c-5034-403c-98c3-5cb7e8bea467 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:17:54.538970Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d13b2-8dc58d59b-hjbgw namespace: kserve-ci-e2e-test resourceVersion: "17468" uid: 717291e3-5be8-4acd-ba06-17079b208344 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-d13b2-8dc58d59b-hjbgw to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-07-27T21:17:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:17:54Z" name: sequence-graph-d13b2-8dc58d59b-hjbgw.18c6422f9b83bf5b namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 578bd78d-a3d5-46aa-9e70-8832146dc696 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:54Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d13b2-8dc58d59b-hjbgw namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: 717291e3-5be8-4acd-ba06-17079b208344 kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-d13b2-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:54Z" name: sequence-graph-d13b2-8dc58d59b-hjbgw.18c6422fa6ba1155 namespace: kserve-ci-e2e-test resourceVersion: "17495" uid: 44943273-acee-4902-bd06-c9d90a12947b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:55Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d13b2-8dc58d59b-hjbgw namespace: kserve-ci-e2e-test resourceVersion: "17477" uid: 717291e3-5be8-4acd-ba06-17079b208344 kind: Event lastTimestamp: "2026-07-27T21:17:55Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:17:55Z" name: sequence-graph-d13b2-8dc58d59b-hjbgw.18c6422fda3e6fae namespace: kserve-ci-e2e-test resourceVersion: "17503" uid: dc8dae50-474d-4516-be9d-257488db83ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d13b2} kind: Pod name: sequence-graph-d13b2-8dc58d59b-hjbgw namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: 717291e3-5be8-4acd-ba06-17079b208344 kind: Event lastTimestamp: "2026-07-27T21:17:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-27T21:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:55Z" name: sequence-graph-d13b2-8dc58d59b-hjbgw.18c6422fdb7ea3b9 namespace: kserve-ci-e2e-test resourceVersion: "17505" uid: 328fc6af-2e59-4b8a-bb73-3280cc44c4e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d13b2} kind: Pod name: sequence-graph-d13b2-8dc58d59b-hjbgw namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: 717291e3-5be8-4acd-ba06-17079b208344 kind: Event lastTimestamp: "2026-07-27T21:17:55Z" message: 'Created container: sequence-graph-d13b2' metadata: creationTimestamp: "2026-07-27T21:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:55Z" name: sequence-graph-d13b2-8dc58d59b-hjbgw.18c6422fdf30160d namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 45c1a27c-eabc-4391-a83a-27edf00a7579 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d13b2} kind: Pod name: sequence-graph-d13b2-8dc58d59b-hjbgw namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: 717291e3-5be8-4acd-ba06-17079b208344 kind: Event lastTimestamp: "2026-07-27T21:17:55Z" message: Started container sequence-graph-d13b2 metadata: creationTimestamp: "2026-07-27T21:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:55Z" name: sequence-graph-d13b2-8dc58d59b-hjbgw.18c6422fdfa7c2b6 namespace: kserve-ci-e2e-test resourceVersion: "17508" uid: 04e6484b-ba9f-4e58-a89c-0b994da8fdaf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d13b2} kind: Pod name: sequence-graph-d13b2-8dc58d59b-hjbgw namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: 717291e3-5be8-4acd-ba06-17079b208344 kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: Stopping container sequence-graph-d13b2 metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:09Z" name: sequence-graph-d13b2-8dc58d59b-hjbgw.18c642a2c38bde16 namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: 461a84da-d0c4-4618-a519-2a7925001749 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d13b2} kind: Pod name: sequence-graph-d13b2-8dc58d59b-hjbgw namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: 717291e3-5be8-4acd-ba06-17079b208344 kind: Event lastTimestamp: "2026-07-27T21:26:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:37Z" name: sequence-graph-d13b2-8dc58d59b-hjbgw.18c642a37cac7ed6 namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 2d276107-4507-4e10-8a86-e4b979e94eee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-d13b2-8dc58d59b namespace: kserve-ci-e2e-test resourceVersion: "17465" uid: 86d1ee40-56b9-4f26-9df4-e09d57987d92 kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: 'Created pod: sequence-graph-d13b2-8dc58d59b-hjbgw' metadata: creationTimestamp: "2026-07-27T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:54Z" name: sequence-graph-d13b2-8dc58d59b.18c6422f9b148911 namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: b9e2d9e6-f9dc-4746-b989-88b8c1bbcb56 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "17463" uid: 7213370b-3977-4947-bf96-40702cc813da kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: Scaled up replica set sequence-graph-d13b2-8dc58d59b from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:54Z" name: sequence-graph-d13b2.18c6422f9a6b2312 namespace: kserve-ci-e2e-test resourceVersion: "17466" uid: 09786f9b-bd57-4622-b335-c3c1083b3da4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "17483" uid: cbc07037-561f-4a84-be36-48f7b01c3838 kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: InferenceGraph [sequence-graph-d13b2] is Ready metadata: creationTimestamp: "2026-07-27T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:54Z" name: sequence-graph-d13b2.18c6422fa02256eb namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: fe92836d-b0a0-4032-bd17-299aaab4393d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:18:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "17464" uid: 94ab1596-1783-4797-b426-7da6855531cc kind: Event lastTimestamp: "2026-07-27T21:18:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:18:09Z" name: sequence-graph-d13b2.18c6423318f7f1ba namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: bca9ba92-c7eb-40a5-8876-bb540234bf6b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:18:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "17464" uid: 94ab1596-1783-4797-b426-7da6855531cc kind: Event lastTimestamp: "2026-07-27T21:18:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:18:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:18:09Z" name: sequence-graph-d13b2.18c64233195d3d59 namespace: kserve-ci-e2e-test resourceVersion: "17613" uid: 3c614ee9-774b-4d97-b32b-8c3293cdedbc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:18:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "17612" uid: 94ab1596-1783-4797-b426-7da6855531cc kind: Event lastTimestamp: "2026-07-27T21:18:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:18:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:18:24Z" name: sequence-graph-d13b2.18c64236988b9e47 namespace: kserve-ci-e2e-test resourceVersion: "17723" uid: 1e26e6df-a89e-4e38-bcfa-6a01a69ac4bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:18:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "17612" uid: 94ab1596-1783-4797-b426-7da6855531cc kind: Event lastTimestamp: "2026-07-27T21:18:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:18:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:18:24Z" name: sequence-graph-d13b2.18c6423698d4d581 namespace: kserve-ci-e2e-test resourceVersion: "17724" uid: a149688c-b218-4bfb-b018-8023ddc78b9a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:37:04.775305Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-61561-79f8d69754-l7866 namespace: kserve-ci-e2e-test resourceVersion: "27041" uid: 2793424c-e52e-4298-bace-85e9689ace53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-61561-79f8d69754-l7866 to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-07-27T21:37:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:37:04Z" name: splitter-graph-61561-79f8d69754-l7866.18c6433b6af150be namespace: kserve-ci-e2e-test resourceVersion: "27049" uid: 12264540-880b-4e1f-85a4-3e26cb2484c2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:05Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-61561-79f8d69754-l7866 namespace: kserve-ci-e2e-test resourceVersion: "27053" uid: 2793424c-e52e-4298-bace-85e9689ace53 kind: Event lastTimestamp: "2026-07-27T21:37:05Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:37:05Z" name: splitter-graph-61561-79f8d69754-l7866.18c6433b84fe262c namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: 24ed4349-d7f7-43b9-adb1-8261fa043d5d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-61561} kind: Pod name: splitter-graph-61561-79f8d69754-l7866 namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: 2793424c-e52e-4298-bace-85e9689ace53 kind: Event lastTimestamp: "2026-07-27T21:37:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-27T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:05Z" name: splitter-graph-61561-79f8d69754-l7866.18c6433b8645d919 namespace: kserve-ci-e2e-test resourceVersion: "27066" uid: 8566e938-255a-4c63-83e8-f11add1b7483 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-61561} kind: Pod name: splitter-graph-61561-79f8d69754-l7866 namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: 2793424c-e52e-4298-bace-85e9689ace53 kind: Event lastTimestamp: "2026-07-27T21:37:05Z" message: 'Created container: splitter-graph-61561' metadata: creationTimestamp: "2026-07-27T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:05Z" name: splitter-graph-61561-79f8d69754-l7866.18c6433b893c61f7 namespace: kserve-ci-e2e-test resourceVersion: "27067" uid: 088c35d5-3dc4-499b-85a1-e5077cc4f6d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-61561} kind: Pod name: splitter-graph-61561-79f8d69754-l7866 namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: 2793424c-e52e-4298-bace-85e9689ace53 kind: Event lastTimestamp: "2026-07-27T21:37:05Z" message: Started container splitter-graph-61561 metadata: creationTimestamp: "2026-07-27T21:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:05Z" name: splitter-graph-61561-79f8d69754-l7866.18c6433b89b5facc namespace: kserve-ci-e2e-test resourceVersion: "27068" uid: e1d4d910-d931-4a64-b44d-74aba122ba84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-61561} kind: Pod name: splitter-graph-61561-79f8d69754-l7866 namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: 2793424c-e52e-4298-bace-85e9689ace53 kind: Event lastTimestamp: "2026-07-27T21:37:14Z" message: Stopping container splitter-graph-61561 metadata: creationTimestamp: "2026-07-27T21:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:14Z" name: splitter-graph-61561-79f8d69754-l7866.18c6433dc390e888 namespace: kserve-ci-e2e-test resourceVersion: "27161" uid: 2d9fe942-9c1d-4046-88c9-a136fce61e64 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-61561} kind: Pod name: splitter-graph-61561-79f8d69754-l7866 namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: 2793424c-e52e-4298-bace-85e9689ace53 kind: Event lastTimestamp: "2026-07-27T21:37:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:41Z" name: splitter-graph-61561-79f8d69754-l7866.18c6433e23d51317 namespace: kserve-ci-e2e-test resourceVersion: "27555" uid: bde7a474-bdac-4d83-af74-3138b0a999ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-61561-79f8d69754 namespace: kserve-ci-e2e-test resourceVersion: "27038" uid: 852345d8-1de9-4596-9d05-2890b03dc74b kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: 'Created pod: splitter-graph-61561-79f8d69754-l7866' metadata: creationTimestamp: "2026-07-27T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: splitter-graph-61561-79f8d69754.18c6433b6a5a6960 namespace: kserve-ci-e2e-test resourceVersion: "27043" uid: 6d7abb6c-4a73-493e-abb3-e06c255c84b7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-61561 namespace: kserve-ci-e2e-test resourceVersion: "27036" uid: c54bf382-d3ae-4d07-862e-f985e2bcd357 kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: Scaled up replica set splitter-graph-61561-79f8d69754 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: splitter-graph-61561.18c6433b69527b76 namespace: kserve-ci-e2e-test resourceVersion: "27039" uid: b7bc027d-0259-4384-91ae-94b3bb9de362 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-61561 namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: d617b2d4-7b66-4ba7-8229-013259dc49af kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: InferenceGraph [splitter-graph-61561] is Ready metadata: creationTimestamp: "2026-07-27T21:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:37:04Z" name: splitter-graph-61561.18c6433b6e8d925a namespace: kserve-ci-e2e-test resourceVersion: "27059" uid: 06e176d6-dfbc-480f-a9fc-b8812c6c39dc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:38:15.086390Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b2e96-c4cf9c888-47dlv namespace: kserve-ci-e2e-test resourceVersion: "27851" uid: 3801fd76-3df5-4ae1-a917-62e1f490c3fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-b2e96-c4cf9c888-47dlv to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-07-27T21:38:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:38:15Z" name: splitter-graph-b2e96-c4cf9c888-47dlv.18c6434bc9cf58dc namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 4ae76f58-89cd-4189-9714-c8d1f1cee9f4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:38:15Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b2e96-c4cf9c888-47dlv namespace: kserve-ci-e2e-test resourceVersion: "27860" uid: 3801fd76-3df5-4ae1-a917-62e1f490c3fe kind: Event lastTimestamp: "2026-07-27T21:38:15Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:38:15Z" name: splitter-graph-b2e96-c4cf9c888-47dlv.18c6434be3afbb0d namespace: kserve-ci-e2e-test resourceVersion: "27872" uid: 01ee1c66-410a-43f9-917e-cb8bac4f9df8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:38:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b2e96} kind: Pod name: splitter-graph-b2e96-c4cf9c888-47dlv namespace: kserve-ci-e2e-test resourceVersion: "27852" uid: 3801fd76-3df5-4ae1-a917-62e1f490c3fe kind: Event lastTimestamp: "2026-07-27T21:38:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-27T21:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:38:15Z" name: splitter-graph-b2e96-c4cf9c888-47dlv.18c6434be4fabb2d namespace: kserve-ci-e2e-test resourceVersion: "27875" uid: 9b749637-2a4c-4a28-afc0-c0235e738f75 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:38:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b2e96} kind: Pod name: splitter-graph-b2e96-c4cf9c888-47dlv namespace: kserve-ci-e2e-test resourceVersion: "27852" uid: 3801fd76-3df5-4ae1-a917-62e1f490c3fe kind: Event lastTimestamp: "2026-07-27T21:38:15Z" message: 'Created container: splitter-graph-b2e96' metadata: creationTimestamp: "2026-07-27T21:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:38:15Z" name: splitter-graph-b2e96-c4cf9c888-47dlv.18c6434be811c88f namespace: kserve-ci-e2e-test resourceVersion: "27876" uid: e79ad921-ea73-4d2b-ae98-e035385ad25f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:38:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b2e96} kind: Pod name: splitter-graph-b2e96-c4cf9c888-47dlv namespace: kserve-ci-e2e-test resourceVersion: "27852" uid: 3801fd76-3df5-4ae1-a917-62e1f490c3fe kind: Event lastTimestamp: "2026-07-27T21:38:15Z" message: Started container splitter-graph-b2e96 metadata: creationTimestamp: "2026-07-27T21:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:38:15Z" name: splitter-graph-b2e96-c4cf9c888-47dlv.18c6434be889302c namespace: kserve-ci-e2e-test resourceVersion: "27877" uid: f92bbf07-ab79-4906-8ab3-0a43eccee029 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:46:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b2e96} kind: Pod name: splitter-graph-b2e96-c4cf9c888-47dlv namespace: kserve-ci-e2e-test resourceVersion: "27852" uid: 3801fd76-3df5-4ae1-a917-62e1f490c3fe kind: Event lastTimestamp: "2026-07-27T21:46:29Z" message: Stopping container splitter-graph-b2e96 metadata: creationTimestamp: "2026-07-27T21:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:46:29Z" name: splitter-graph-b2e96-c4cf9c888-47dlv.18c643bef1b69bf7 namespace: kserve-ci-e2e-test resourceVersion: "31299" uid: 601a6e35-a343-49ec-bb48-4a7612674777 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b2e96} kind: Pod name: splitter-graph-b2e96-c4cf9c888-47dlv namespace: kserve-ci-e2e-test resourceVersion: "27852" uid: 3801fd76-3df5-4ae1-a917-62e1f490c3fe kind: Event lastTimestamp: "2026-07-27T21:46:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:46:56Z" name: splitter-graph-b2e96-c4cf9c888-47dlv.18c643bf6f66a324 namespace: kserve-ci-e2e-test resourceVersion: "31529" uid: a592a8aa-e85c-4e0b-b3db-7250f4bced93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:38:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-b2e96-c4cf9c888 namespace: kserve-ci-e2e-test resourceVersion: "27846" uid: 87cd2e65-c651-45ea-8dc1-d9b7eefe895d kind: Event lastTimestamp: "2026-07-27T21:38:15Z" message: 'Created pod: splitter-graph-b2e96-c4cf9c888-47dlv' metadata: creationTimestamp: "2026-07-27T21:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:15Z" name: splitter-graph-b2e96-c4cf9c888.18c6434bc96b96a6 namespace: kserve-ci-e2e-test resourceVersion: "27853" uid: 6a26d4fa-a5e7-4ba5-a30e-aedb513c8220 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:38:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27844" uid: 5e49067d-4406-4b6e-abfa-84ac210dd46c kind: Event lastTimestamp: "2026-07-27T21:38:15Z" message: Scaled up replica set splitter-graph-b2e96-c4cf9c888 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:15Z" name: splitter-graph-b2e96.18c6434bc88567c0 namespace: kserve-ci-e2e-test resourceVersion: "27848" uid: 5a2c1bca-fe1f-41e4-86db-b11cb34f2366 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:38:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27862" uid: 9881a866-c22a-453a-a8b6-76c382521778 kind: Event lastTimestamp: "2026-07-27T21:38:15Z" message: InferenceGraph [splitter-graph-b2e96] is Ready metadata: creationTimestamp: "2026-07-27T21:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:38:15Z" name: splitter-graph-b2e96.18c6434bcd9abe8d namespace: kserve-ci-e2e-test resourceVersion: "27865" uid: 947b6da3-e32e-4389-ac52-88f1e7895b38 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:38:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27845" uid: f3332bd8-5f6e-4108-a167-25037fa346ca kind: Event lastTimestamp: "2026-07-27T21:38:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:45Z" name: splitter-graph-b2e96.18c6434f47098cf6 namespace: kserve-ci-e2e-test resourceVersion: "28090" uid: 08e50c70-13d2-4da7-9878-507dd57d422b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:38:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27845" uid: f3332bd8-5f6e-4108-a167-25037fa346ca kind: Event lastTimestamp: "2026-07-27T21:38:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:45Z" name: splitter-graph-b2e96.18c6434f476f9d93 namespace: kserve-ci-e2e-test resourceVersion: "28091" uid: be77e81d-4147-4fff-a999-87185c0060d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:14:34.500450Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-070ff-predictor-6488bc79fc-4ksmq to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:14:34Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c64201084a2c62 namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: dd6f64c2-d423-4e38-84a0-42894f920ed2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-070ff-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:35Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c64201155e3137 namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: f4c1e2d3-7cc0-4c0c-891d-8256cf796203 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:36Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:14:36Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c642017b9a758d namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: cafb16be-65f0-44fc-9e67-fee7ef5c078a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:36Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-27T21:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:36Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c642017ced4cd6 namespace: kserve-ci-e2e-test resourceVersion: "15278" uid: 6486abad-cfe1-4556-91de-608e1b9f6879 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:53Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" in 17.08s (17.08s including waiting). Image size: 1334928497 bytes.' metadata: creationTimestamp: "2026-07-27T21:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:53Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c6420576fc4297 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: b0b717f6-e93c-49f1-b378-850edf9342a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:54Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c642059db7e535 namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 669cc7ec-91f9-43ed-8305-764fd525dbff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:54Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c642059e3aa19e namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 92ef8ea7-bf19-4e05-8ee1-2b7f02395b33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-27T21:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:54Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c642059e4b5ca2 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 0d1facb5-12f4-494d-8332-88c4946f23b0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:57Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.48s (3.48s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:57Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c642066dbd5ff5 namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 37358e6e-f358-4fc9-9250-a0fe3f149aa1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:57Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c64206735837f7 namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: f0ae418d-197f-49b3-8344-aba2c50cb766 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:14:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:14:57Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c64206740804ce namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 6d52d1fc-f6f8-4200-8a36-2cacbf8c18da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-27T21:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c64206b0d44b8b namespace: kserve-ci-e2e-test resourceVersion: "16395" uid: bf31e65a-7fe1-4392-bab5-a54e6e372f47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c6421aaa09b62c namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 27454fc7-c7b6-403a-8d51-bec68ed576a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c6421aaa0a2b0f namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: bfd368b9-ea9e-477a-b79b-8a20f356333a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c93187fa-dfa6-4ab5-a0fc-8d66d8f34e1e kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq.18c6421ab756b08e namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: bd58988a-d8af-4da1-9c16-4b9b57a93425 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-070ff-predictor-6488bc79fc namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: 4bce2ed6-f884-4322-997d-7b081f8fbace kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: 'Created pod: success-200-isvc-070ff-predictor-6488bc79fc-4ksmq' metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:34Z" name: success-200-isvc-070ff-predictor-6488bc79fc.18c6420107e2728f namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: f598dfa1-734a-4009-8e23-52147cb0b572 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: 9db76c1f-aa6e-4c7d-a891-89505abb2d32 kind: Event lastTimestamp: "2026-07-27T21:14:34Z" message: Scaled up replica set success-200-isvc-070ff-predictor-6488bc79fc from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:14:34Z" name: success-200-isvc-070ff-predictor.18c642010488d7cf namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 743b7d4d-1b0d-4801-9c73-4bdd51e2ac3a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: 22acf2f1-6992-4f4a-9ed6-87c2413d3e30 kind: Event lastTimestamp: "2026-07-27T21:15:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:34Z" name: success-200-isvc-070ff-predictor.18c6420482fd7493 namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 6a1fa04b-f7fd-4307-8db0-da8b0d098f0e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:14:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: 22acf2f1-6992-4f4a-9ed6-87c2413d3e30 kind: Event lastTimestamp: "2026-07-27T21:15:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:15:34Z" name: success-200-isvc-070ff-predictor.18c64204835defef namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 306b6fc1-b463-4de8-b820-996bcaa995a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 22acf2f1-6992-4f4a-9ed6-87c2413d3e30 kind: Event lastTimestamp: "2026-07-27T21:16:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:19Z" name: success-200-isvc-070ff-predictor.18c6420b80bcc2f8 namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: 2b66ddf3-dc4d-4f02-b410-273b9bdb128b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:15:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-070ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 22acf2f1-6992-4f4a-9ed6-87c2413d3e30 kind: Event lastTimestamp: "2026-07-27T21:16:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:15:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:19Z" name: success-200-isvc-070ff-predictor.18c6420b8128d7dc namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 5f2656b4-0ea9-499e-8bca-5a8829b48eed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:14:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-070ff namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: dff93d3c-7d0f-4af5-adbf-18309548a15a kind: Event lastTimestamp: "2026-07-27T21:15:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:15:54Z" name: success-200-isvc-070ff.18c64201048a1216 namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: fe2bf87f-5a8a-4f75-acd9-b3a4382b1335 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:15:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-070ff namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: dff93d3c-7d0f-4af5-adbf-18309548a15a kind: Event lastTimestamp: "2026-07-27T21:15:54Z" message: InferenceService [success-200-isvc-070ff] is Ready metadata: creationTimestamp: "2026-07-27T21:15:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:15:54Z" name: success-200-isvc-070ff.18c64213c2628798 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 86c375b8-a4c2-4e3b-a27e-a9f07c6ae345 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:26:09.295987Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:26:09Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a2cd54bdd9 namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: 3493ce81-9083-4308-8ed9-13849ce36dbf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0a095-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:09Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a2d7ca94f1 namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: 38fd8088-3ab6-424a-aea3-f682d927667a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:26:10Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a30a8565ce namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: ccdb44e2-fa86-46e7-9f42-d8abe949ab54 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a30c42b5c5 namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: 658ede27-1a60-4497-b47f-500212c09749 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a30f80931a namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 9bc392ca-5544-4337-813a-fb937ca74339 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a30ff577c8 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 88f65e3d-375d-4dee-91b1-5bf3922f91ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a30fff8051 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 72f6e9b5-1130-4be8-b56d-30b18e840d05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a312f02564 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 649c3f13-ea34-4e4e-a608-0822bf4fa2f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:26:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:10Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a31363ca09 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 8b136b95-84f4-40da-9ad5-a31981b16644 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:26:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:46Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642a323ef5f9c namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: a5f5eb96-2e4d-4f79-b043-868583a3fbe4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:19Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642b325c718b2 namespace: kserve-ci-e2e-test resourceVersion: "22073" uid: 193d9cbd-9398-482b-8f9e-3c004612d51e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:19Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642b325c7c6d6 namespace: kserve-ci-e2e-test resourceVersion: "22076" uid: f41d32eb-4191-4c27-85b6-1a1f0fb0643e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e4098812-fad8-4744-bf13-000e088a6126 kind: Event lastTimestamp: "2026-07-27T21:27:21Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:21Z" name: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5.18c642b3ac186967 namespace: kserve-ci-e2e-test resourceVersion: "22196" uid: 549fdace-d5c8-4715-a986-27546e1119a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0a095-predictor-7c5d79bf89 namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: e9ca3293-540f-40b3-ab73-d8b76ace95ab kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: 'Created pod: success-200-isvc-0a095-predictor-7c5d79bf89-bfbs5' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:09Z" name: success-200-isvc-0a095-predictor-7c5d79bf89.18c642a2cce8ddc5 namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: abcfa976-0a2d-4216-a0a4-839a9d50d023 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: f28eff73-17c6-41a7-855b-74bd70becef2 kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: Scaled up replica set success-200-isvc-0a095-predictor-7c5d79bf89 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:09Z" name: success-200-isvc-0a095-predictor.18c642a2cb6c90c3 namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: a1701c3a-a9fa-49cf-8750-4deb26dee424 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: 8b7b5007-6766-448a-998d-09ef7d0c7b09 kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: success-200-isvc-0a095-predictor.18c642a64a543e5f namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: 381b5b9b-aabf-4560-92b4-7ffd2b2bf2ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:26:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: 8b7b5007-6766-448a-998d-09ef7d0c7b09 kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: success-200-isvc-0a095-predictor.18c642a64ab35a6e namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 5dbbeeb3-0b75-4d8c-86c8-52c80a2f400e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:26:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: 8b7b5007-6766-448a-998d-09ef7d0c7b09 kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:26:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:09Z" name: success-200-isvc-0a095-predictor.18c642ad48177ef5 namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a46ad00f-a1c5-46d4-b762-57748d7b2275 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:26:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a095-predictor namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: 8b7b5007-6766-448a-998d-09ef7d0c7b09 kind: Event lastTimestamp: "2026-07-27T21:27:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:26:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:09Z" name: success-200-isvc-0a095-predictor.18c642ad487e445c namespace: kserve-ci-e2e-test resourceVersion: "21938" uid: 691df907-7682-477e-8105-f2ce72b5e478 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 9b339c07-b9b2-47f5-97e0-ac645efc2d1b kind: Event lastTimestamp: "2026-07-27T21:26:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:56Z" name: success-200-isvc-0a095.18c642a2cb7eba5a namespace: kserve-ci-e2e-test resourceVersion: "21809" uid: 93bd496c-7baa-4494-8519-9998f62c918e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 9b339c07-b9b2-47f5-97e0-ac645efc2d1b kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: 'Failed to update status for InferenceService "success-200-isvc-0a095": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0a095": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:09Z" name: success-200-isvc-0a095.18c642a2cbe4ee15 namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: af98f33f-c388-47a4-b7f1-0c80fd8a3ccd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 9b339c07-b9b2-47f5-97e0-ac645efc2d1b kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0a095": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:09Z" name: success-200-isvc-0a095.18c642a2cbe549c9 namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: be4fed91-ad10-40f5-8c7f-c642015fac8b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a095 namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 9b339c07-b9b2-47f5-97e0-ac645efc2d1b kind: Event lastTimestamp: "2026-07-27T21:26:56Z" message: InferenceService [success-200-isvc-0a095] is Ready metadata: creationTimestamp: "2026-07-27T21:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:56Z" name: success-200-isvc-0a095.18c642addf230bdd namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: 047817d0-574d-4daa-a39e-66bb3cc2a720 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:27:19.567850Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22099" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:27:19Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b329dc44d8 namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: 248430af-cf47-401b-ae34-7c91d7c0b450 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0f5d7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:19Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b336e3bd26 namespace: kserve-ci-e2e-test resourceVersion: "22145" uid: 817605b5-0fca-4229-abc7-12875d54650d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:27:20Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b3670a4eee namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 807a0ecb-1472-458e-908c-71e388c501eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b36874c362 namespace: kserve-ci-e2e-test resourceVersion: "22176" uid: b95597e9-0c65-4983-9530-42e9ed3f4c3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b36c0346de namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: c7dcae6e-c4c7-499d-938b-0f49b64baf05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b36cae8973 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: daf39c49-7462-42d0-a391-b21fb1a681dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b36cb80313 namespace: kserve-ci-e2e-test resourceVersion: "22179" uid: 11d89a23-7dcc-440f-9619-84e10f9f8a7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b36f6043c5 namespace: kserve-ci-e2e-test resourceVersion: "22180" uid: 6b5f5ec2-dad3-4ec1-8f49-cebf1f3dff92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:27:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:20Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b36fe8eda6 namespace: kserve-ci-e2e-test resourceVersion: "22182" uid: adbe8201-23f3-483d-90f1-453648801e5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:27:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:27:57Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c642b383c7e8e1 namespace: kserve-ci-e2e-test resourceVersion: "22542" uid: 8531316c-de6a-4b28-b10f-4db83fa64118 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:34Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c6433459347734 namespace: kserve-ci-e2e-test resourceVersion: "26669" uid: b2641a5f-58ce-4351-9763-6f3073d47a4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: d4f550dd-99e5-4ddd-b84c-e89d1af94c2b kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:34Z" name: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7.18c64334593587a7 namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: 11fb417f-6e89-44bc-b9fb-e96f4074dc54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0f5d7-predictor-69c54f994c namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: af70f709-3bcc-46e3-9e8b-73bdc19867c4 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: 'Created pod: success-200-isvc-0f5d7-predictor-69c54f994c-99rb7' metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:19Z" name: success-200-isvc-0f5d7-predictor-69c54f994c.18c642b329702fa4 namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: 8fc72d8f-dce7-49ad-b288-77480b90630f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 80482732-0a5e-4eb3-abf9-0eaeb0cf76cf kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: Scaled up replica set success-200-isvc-0f5d7-predictor-69c54f994c from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:19Z" name: success-200-isvc-0f5d7-predictor.18c642b32717d76e namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: 9e1c1d26-693a-405e-8f2f-2b383d4105e4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: b868f2c5-7e40-46df-9c0d-a6f284c2d2e6 kind: Event lastTimestamp: "2026-07-27T21:27:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:49Z" name: success-200-isvc-0f5d7-predictor.18c642b6a5a93696 namespace: kserve-ci-e2e-test resourceVersion: "22439" uid: 3cbff856-b6b2-45bc-8579-87c0d099029c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: b868f2c5-7e40-46df-9c0d-a6f284c2d2e6 kind: Event lastTimestamp: "2026-07-27T21:27:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:49Z" name: success-200-isvc-0f5d7-predictor.18c642b6a611d5c0 namespace: kserve-ci-e2e-test resourceVersion: "22440" uid: f1894cce-6a5a-4bec-9e00-4723e709f1b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:28:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22300" uid: b868f2c5-7e40-46df-9c0d-a6f284c2d2e6 kind: Event lastTimestamp: "2026-07-27T21:28:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:34Z" name: success-200-isvc-0f5d7-predictor.18c642bda40626f4 namespace: kserve-ci-e2e-test resourceVersion: "22879" uid: e851e954-c893-4da6-a3f5-23da46727475 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:28:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0f5d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22300" uid: b868f2c5-7e40-46df-9c0d-a6f284c2d2e6 kind: Event lastTimestamp: "2026-07-27T21:28:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:34Z" name: success-200-isvc-0f5d7-predictor.18c642bda4712ca1 namespace: kserve-ci-e2e-test resourceVersion: "22880" uid: 8215fde5-1381-4343-96a8-d81ffda1c071 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:27:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: fe1098c0-e768-497e-9923-18430097913a kind: Event lastTimestamp: "2026-07-27T21:28:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:28:07Z" name: success-200-isvc-0f5d7.18c642b32763e81e namespace: kserve-ci-e2e-test resourceVersion: "22630" uid: acaf143d-a2f5-4582-8c6c-7ebceb426eca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:28:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0f5d7 namespace: kserve-ci-e2e-test resourceVersion: "22627" uid: fe1098c0-e768-497e-9923-18430097913a kind: Event lastTimestamp: "2026-07-27T21:28:07Z" message: InferenceService [success-200-isvc-0f5d7] is Ready metadata: creationTimestamp: "2026-07-27T21:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:28:07Z" name: success-200-isvc-0f5d7.18c642be41c77d53 namespace: kserve-ci-e2e-test resourceVersion: "22628" uid: bfba47c7-9753-4c4e-a63a-abe4e1acd683 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:36:04.700879Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:36:04Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432d6e3a6121 namespace: kserve-ci-e2e-test resourceVersion: "26300" uid: 168bbf2f-cf26-4b5a-b15d-66811f0c3830 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-61561-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:04Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432d789283f8 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: d3214a3e-e7d3-49b5-9468-77d33ea089bc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:36:05Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432dab1e895a namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 32ebeb1f-07be-4c34-b332-71ec80d3a6c2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432dac75a6b3 namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: b7292e99-a7f7-4dbe-b4da-a0f22282cb95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432db01b78d7 namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: aa54ad36-728d-49d8-8cba-08ffe0909eda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432db089aee0 namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: 9154a337-227c-447e-b8ae-5a3231d7fa72 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432db093348d namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 3ac93c51-a628-4894-8787-624c7e01f933 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432db3a3a5e9 namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: b3924fef-0598-4d36-abcf-161976113b70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:36:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:05Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432db411fd94 namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 1f6a6a40-2b0f-4dbb-b99e-9ff27d76e114 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:36:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:43Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6432e0883cb2b namespace: kserve-ci-e2e-test resourceVersion: "26836" uid: ea4c67bb-66aa-49f5-aac6-c473fb62a64f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:37:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:14Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6433dca3de5c8 namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: e040ee54-af4d-43c3-ab92-c18cbe88bc4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:37:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:14Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6433dca3e7150 namespace: kserve-ci-e2e-test resourceVersion: "27185" uid: da75d5be-be76-4aff-9cd5-af4c364a3192 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: ce42dd03-8598-4512-9162-bc0232e54d77 kind: Event lastTimestamp: "2026-07-27T21:37:19Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-27T21:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:19Z" name: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6.18c6433ecc5fc9db namespace: kserve-ci-e2e-test resourceVersion: "27322" uid: 4ee38307-6255-4514-b609-ba9f32b22d0b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-61561-predictor-6959dd4cf7 namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: 972a6250-6c5a-4699-9ccd-026b9277c34c kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: 'Created pod: success-200-isvc-61561-predictor-6959dd4cf7-jjqv6' metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:04Z" name: success-200-isvc-61561-predictor-6959dd4cf7.18c6432d6dd686e5 namespace: kserve-ci-e2e-test resourceVersion: "26294" uid: c93fcc1a-2efb-42d7-87a5-8f9ad09d1ef4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 4d6b56ae-2dfb-43c6-8542-3a230adad44a kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: Scaled up replica set success-200-isvc-61561-predictor-6959dd4cf7 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:04Z" name: success-200-isvc-61561-predictor.18c6432d6a3986c5 namespace: kserve-ci-e2e-test resourceVersion: "26281" uid: bbdd2e52-080c-4215-a636-37c180568d53 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: aad42053-25aa-40ed-8692-9f4d51fb2c36 kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:34Z" name: success-200-isvc-61561-predictor.18c64330e9191a1e namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: a46b05d5-e019-4868-b5eb-12fe463a91d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:36:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: aad42053-25aa-40ed-8692-9f4d51fb2c36 kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:36:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:34Z" name: success-200-isvc-61561-predictor.18c64330e98654b2 namespace: kserve-ci-e2e-test resourceVersion: "26726" uid: 7fde1690-5386-4915-a892-148ffafe422b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:36:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: aad42053-25aa-40ed-8692-9f4d51fb2c36 kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:36:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: success-200-isvc-61561-predictor.18c64337e709e395 namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: c17fc51e-5ddb-46a1-a895-81c8e32771b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:36:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-61561-predictor namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: aad42053-25aa-40ed-8692-9f4d51fb2c36 kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:36:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: success-200-isvc-61561-predictor.18c64337e786862b namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 2b7ad1af-ee06-437d-87da-8ebc6c3da1d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-61561 namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 62a7df28-c471-4570-a728-1890e761bde7 kind: Event lastTimestamp: "2026-07-27T21:36:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:53Z" name: success-200-isvc-61561.18c6432d6a3f0700 namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: e1d592a7-7a50-4299-8082-894f43aa230b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-61561 namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 62a7df28-c471-4570-a728-1890e761bde7 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-61561": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-61561": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:04Z" name: success-200-isvc-61561.18c6432d6b14d70f namespace: kserve-ci-e2e-test resourceVersion: "26287" uid: 979ded20-0f6b-432d-bf3d-f252609ef66d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-61561 namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 62a7df28-c471-4570-a728-1890e761bde7 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-61561": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:04Z" name: success-200-isvc-61561.18c6432d6b159f87 namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: bab2a26e-e54e-4801-98e9-d6231abd0de1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-61561 namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: 62a7df28-c471-4570-a728-1890e761bde7 kind: Event lastTimestamp: "2026-07-27T21:36:53Z" message: InferenceService [success-200-isvc-61561] is Ready metadata: creationTimestamp: "2026-07-27T21:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:53Z" name: success-200-isvc-61561.18c64338c3a2fa21 namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: 01876f0f-1104-46e7-a7b4-e1b33ccbae89 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:37:15.027825Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27207" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:37:15Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433dce0a69d0 namespace: kserve-ci-e2e-test resourceVersion: "27213" uid: ab7d8a82-982c-4ee1-9568-e57ecc66e812 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b2e96-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:15Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433dd82b064e namespace: kserve-ci-e2e-test resourceVersion: "27252" uid: 0fd09828-0e61-4d4b-ab9b-7c629432e4d6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27222" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:37:16Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:37:16Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433e0b15f8e0 namespace: kserve-ci-e2e-test resourceVersion: "27276" uid: 4f0c5a5c-00a3-4967-9140-98a1b230e801 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:37:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:16Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433e0c65b180 namespace: kserve-ci-e2e-test resourceVersion: "27278" uid: 3980d7ae-a6fc-4c87-84e9-c8036180d7d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:37:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:16Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433e0f5e50e8 namespace: kserve-ci-e2e-test resourceVersion: "27279" uid: 963d6d2e-a49c-4283-b88c-949e5f58340a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:37:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:16Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433e0fdd0f2a namespace: kserve-ci-e2e-test resourceVersion: "27280" uid: d5bdccdc-6d27-4f57-b0d2-162282093fce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:37:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:16Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433e0fe5adcd namespace: kserve-ci-e2e-test resourceVersion: "27281" uid: 794aa7c4-05c7-40de-9a0b-93c5cdd779be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:37:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:16Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433e131f371f namespace: kserve-ci-e2e-test resourceVersion: "27284" uid: 750d0c99-493b-42e8-b6ee-7e59e3d73fa2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:37:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:16Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433e1395bea5 namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: 48f94bbc-c3dc-47c1-a3f6-f40c31b6db71 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:37:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:37:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:53Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c6433e675937d4 namespace: kserve-ci-e2e-test resourceVersion: "27661" uid: 58f025de-ede3-4479-9e57-faefef93e97d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:46:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:46:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:46:29Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c643bef963757f namespace: kserve-ci-e2e-test resourceVersion: "31317" uid: bc10a08f-47e8-4249-8277-734d1eb4e7cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:46:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 77479cc9-99d5-49a3-b91a-ccdec2877ff3 kind: Event lastTimestamp: "2026-07-27T21:46:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:46:29Z" name: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48.18c643bef9644277 namespace: kserve-ci-e2e-test resourceVersion: "31318" uid: 299ed2dc-d881-4887-a524-a3dcf7b786b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b2e96-predictor-c4576f9f9 namespace: kserve-ci-e2e-test resourceVersion: "27191" uid: e3a0526b-d901-4ffc-8ac6-55fa36f952db kind: Event lastTimestamp: "2026-07-27T21:37:15Z" message: 'Created pod: success-200-isvc-b2e96-predictor-c4576f9f9-9xl48' metadata: creationTimestamp: "2026-07-27T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:15Z" name: success-200-isvc-b2e96-predictor-c4576f9f9.18c6433dcd9ebd43 namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 8c36a7f0-8210-4591-a978-b65888a67ce8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27187" uid: f56eef48-e113-4d67-a470-1dcd07128967 kind: Event lastTimestamp: "2026-07-27T21:37:14Z" message: Scaled up replica set success-200-isvc-b2e96-predictor-c4576f9f9 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:14Z" name: success-200-isvc-b2e96-predictor.18c6433dcba6dfd2 namespace: kserve-ci-e2e-test resourceVersion: "27197" uid: 86534d95-55f7-48ef-a238-c819285f0db5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27190" uid: 58de7e19-47c6-4bc9-84a2-44374908a2fc kind: Event lastTimestamp: "2026-07-27T21:37:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:45Z" name: success-200-isvc-b2e96-predictor.18c643414a39151a namespace: kserve-ci-e2e-test resourceVersion: "27588" uid: 81502468-5a66-48da-9edc-439a7c28bc9a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:37:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27190" uid: 58de7e19-47c6-4bc9-84a2-44374908a2fc kind: Event lastTimestamp: "2026-07-27T21:37:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:45Z" name: success-200-isvc-b2e96-predictor.18c643414a9f207c namespace: kserve-ci-e2e-test resourceVersion: "27590" uid: 11269389-2bb1-4d1f-b31d-8f1a298f2ae3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:38:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27429" uid: 58de7e19-47c6-4bc9-84a2-44374908a2fc kind: Event lastTimestamp: "2026-07-27T21:38:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:15Z" name: success-200-isvc-b2e96-predictor.18c643484874df6a namespace: kserve-ci-e2e-test resourceVersion: "27847" uid: 03066838-475b-4f95-904a-05df991e3886 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:38:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e96-predictor namespace: kserve-ci-e2e-test resourceVersion: "27429" uid: 58de7e19-47c6-4bc9-84a2-44374908a2fc kind: Event lastTimestamp: "2026-07-27T21:38:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:15Z" name: success-200-isvc-b2e96-predictor.18c6434848da9521 namespace: kserve-ci-e2e-test resourceVersion: "27850" uid: 221d330c-ec11-4083-bcfa-21d5cdeb267b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27166" uid: 794c31eb-6b22-43b6-8ac5-9ee3bf151559 kind: Event lastTimestamp: "2026-07-27T21:38:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:38:03Z" name: success-200-isvc-b2e96.18c6433dcbfdea09 namespace: kserve-ci-e2e-test resourceVersion: "27754" uid: 5d4badfd-0f7a-4f3a-8fae-751106a39594 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:38:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e96 namespace: kserve-ci-e2e-test resourceVersion: "27750" uid: 794c31eb-6b22-43b6-8ac5-9ee3bf151559 kind: Event lastTimestamp: "2026-07-27T21:38:03Z" message: InferenceService [success-200-isvc-b2e96] is Ready metadata: creationTimestamp: "2026-07-27T21:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:38:03Z" name: success-200-isvc-b2e96.18c643492240ce80 namespace: kserve-ci-e2e-test resourceVersion: "27751" uid: feff52ae-981c-4109-92ee-3b896b30a83b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:36:34.463710Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26680" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-beb6a-predictor-d9b4c445f-kbknm to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:36:34Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c643345c3b25d1 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 1240f527-8a4d-4726-ab10-e692d1b80bd7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:36:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-beb6a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:35Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c643346a7dd602 namespace: kserve-ci-e2e-test resourceVersion: "26745" uid: 2db85585-7b85-48f9-9d60-009a0949ad7b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:36:36Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:36:36Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c64334ced824d2 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 0811bf7b-079e-46d9-8b9a-2c530750c1a8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:36:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:36Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c64334d022c119 namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: f011491c-e768-4925-a1aa-238a11e90513 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:36:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:36Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c64334d3470046 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 0254faf8-797b-4189-a736-d9b9fcba3233 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:36:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:36Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c64334d3b0fd8b namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: 3fcd06bc-08a9-462d-aed4-c296d567fb3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:36:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:36Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c64334d3ba51aa namespace: kserve-ci-e2e-test resourceVersion: "26770" uid: f078bec0-d9e9-4119-94d3-63bc3f05c739 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:36:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:36Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c64334d6820a6c namespace: kserve-ci-e2e-test resourceVersion: "26771" uid: 85451392-38ef-4192-b8bb-d07657afcdeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:36:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:36Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c64334d7017a5a namespace: kserve-ci-e2e-test resourceVersion: "26773" uid: 4299e6fa-f883-4f89-a038-20854580e68e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:37:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:13Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c643350c830fd5 namespace: kserve-ci-e2e-test resourceVersion: "27134" uid: ee127402-a495-43e0-9ecc-eac2573e8b01 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:53:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:53:53Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c64426553845e3 namespace: kserve-ci-e2e-test resourceVersion: "34403" uid: f0577513-162f-4503-9a01-69420e7951a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: a10c2012-9cac-49ee-9d83-fdccde059d06 kind: Event lastTimestamp: "2026-07-27T21:53:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:53:53Z" name: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm.18c6442655394f56 namespace: kserve-ci-e2e-test resourceVersion: "34404" uid: a37343e6-f6bc-4b2f-bb31-fa3abda33e8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-beb6a-predictor-d9b4c445f namespace: kserve-ci-e2e-test resourceVersion: "26671" uid: cb712144-2db8-4b64-8d50-7198c38f0731 kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: 'Created pod: success-200-isvc-beb6a-predictor-d9b4c445f-kbknm' metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:34Z" name: success-200-isvc-beb6a-predictor-d9b4c445f.18c643345bd19d0d namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 1b6c3b9e-db58-41af-896a-3acee4625229 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: 868ebfc7-f649-4352-ad4c-023459a3241a kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: Scaled up replica set success-200-isvc-beb6a-predictor-d9b4c445f from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:36:34Z" name: success-200-isvc-beb6a-predictor.18c6433459e5d0bf namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 6096238f-f527-4877-b41d-01e1d70af1ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:36:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 7d774e19-bd0e-4d21-82e9-9793c50a49e2 kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:36:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: success-200-isvc-beb6a-predictor.18c64337d8d0bc4d namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: f80b6395-e615-4fb4-90b9-061e0bd965ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:36:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 7d774e19-bd0e-4d21-82e9-9793c50a49e2 kind: Event lastTimestamp: "2026-07-27T21:37:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:36:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:04Z" name: success-200-isvc-beb6a-predictor.18c64337d91510b7 namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 3c1f20d8-9266-4e4a-ba42-00baa8005d6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:37:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: 7d774e19-bd0e-4d21-82e9-9793c50a49e2 kind: Event lastTimestamp: "2026-07-27T21:37:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:49Z" name: success-200-isvc-beb6a-predictor.18c6433ed6bcfb17 namespace: kserve-ci-e2e-test resourceVersion: "27624" uid: 6d611686-8ee6-4a56-9b5b-11ab8ad6639b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:37:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-beb6a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: 7d774e19-bd0e-4d21-82e9-9793c50a49e2 kind: Event lastTimestamp: "2026-07-27T21:37:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:49Z" name: success-200-isvc-beb6a-predictor.18c6433ed72c46f8 namespace: kserve-ci-e2e-test resourceVersion: "27625" uid: d62ce75c-220a-466e-a553-f6ac6ad6c535 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-beb6a namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: fddafab9-f22a-4849-b41b-3335b4a7d13d kind: Event lastTimestamp: "2026-07-27T21:37:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:37:23Z" name: success-200-isvc-beb6a.18c6433459e47008 namespace: kserve-ci-e2e-test resourceVersion: "27381" uid: b83dd46a-c424-421e-84b5-fd6c3c9c3f52 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-beb6a namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: fddafab9-f22a-4849-b41b-3335b4a7d13d kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: 'Failed to update status for InferenceService "success-200-isvc-beb6a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-beb6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:34Z" name: success-200-isvc-beb6a.18c643345aba1bba namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 5bb59396-cb61-4b6a-8193-f4629836199b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-beb6a namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: fddafab9-f22a-4849-b41b-3335b4a7d13d kind: Event lastTimestamp: "2026-07-27T21:36:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-beb6a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:36:34Z" name: success-200-isvc-beb6a.18c643345aba6a69 namespace: kserve-ci-e2e-test resourceVersion: "26679" uid: fd0ddf14-9b3f-497e-be6d-56ae15ff7212 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-beb6a namespace: kserve-ci-e2e-test resourceVersion: "27377" uid: fddafab9-f22a-4849-b41b-3335b4a7d13d kind: Event lastTimestamp: "2026-07-27T21:37:23Z" message: InferenceService [success-200-isvc-beb6a] is Ready metadata: creationTimestamp: "2026-07-27T21:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:37:23Z" name: success-200-isvc-beb6a.18c6433fc81c4102 namespace: kserve-ci-e2e-test resourceVersion: "27379" uid: 075b549c-23e1-4584-b8ef-3b5eefd10fe8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:25:39.582209Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cb683-predictor-5944ff6487-6sbjs to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:25:39Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c6429be2407ea3 namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 582357da-1ee9-4879-a0bb-3d6c2bd0fe6c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:25:40Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c6429bfc14e5a8 namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 3aa677e0-9292-4371-9a43-8f605e7e66f5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c6429bfd7475f5 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 14ec9cbe-6023-4f8b-9061-5ddff8c4f974 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c6429c00274927 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: f8a04f49-52c3-4c1c-9470-b2e264ff3203 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c6429c0099413f namespace: kserve-ci-e2e-test resourceVersion: "20818" uid: b66b1d27-2753-40ef-a617-d5cb3bb5a9f7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c6429c00a3923f namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 1143a5b9-f510-41c4-abc6-b1612d00d6fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c6429c04986ab1 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 0cd66034-ab0a-463b-adc3-903462629f0e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:25:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:40Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c6429c050e1ff5 namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: 5267da85-0e2e-489d-8cde-193a7ecd2af7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:25:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:26:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:17Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c6429c5bcd70b4 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 43a5a0d0-2f33-4401-b316-520dddfd3c09 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:49Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c642ac3ee3b14a namespace: kserve-ci-e2e-test resourceVersion: "21667" uid: 3624225c-b610-4d8b-a122-1dd5013f71e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:49Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c642ac3ee430b6 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: 49b8baeb-872b-4c08-8ef7-363f3a86eb32 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: ca42de00-87a9-496e-b072-944b45209902 kind: Event lastTimestamp: "2026-07-27T21:26:52Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:52Z" name: success-200-isvc-cb683-predictor-5944ff6487-6sbjs.18c642ace3f1aa10 namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: 92530498-33a5-48fc-8692-bb994dc951c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cb683-predictor-5944ff6487 namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 87af89aa-4abc-4fe8-848f-b3a08c1b2073 kind: Event lastTimestamp: "2026-07-27T21:25:39Z" message: 'Created pod: success-200-isvc-cb683-predictor-5944ff6487-6sbjs' metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:25:39Z" name: success-200-isvc-cb683-predictor-5944ff6487.18c6429be1dbd19d namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 1992cd3f-743f-491e-8fd9-61cf302e7e52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 2b4aa2fc-e832-462c-b979-8cbe87839e65 kind: Event lastTimestamp: "2026-07-27T21:25:39Z" message: Scaled up replica set success-200-isvc-cb683-predictor-5944ff6487 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:25:39Z" name: success-200-isvc-cb683-predictor.18c6429bde5ca957 namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 52835906-6b1d-4184-883c-f833b334dc56 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:25:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 5f544c04-0540-490d-a977-518ce47ce40f kind: Event lastTimestamp: "2026-07-27T21:26:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:24Z" name: success-200-isvc-cb683-predictor.18c6429f5d7034a2 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: da336600-41b9-4bd5-a68a-97dcb31c182a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:25:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 5f544c04-0540-490d-a977-518ce47ce40f kind: Event lastTimestamp: "2026-07-27T21:26:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:24Z" name: success-200-isvc-cb683-predictor.18c6429f5dd8585b namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 552ee0e5-4c0f-44ec-86d9-c42e8f4976ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 5f544c04-0540-490d-a977-518ce47ce40f kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: success-200-isvc-cb683-predictor.18c642a9d9e738c7 namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: 39cc5744-6af6-4f1e-b1bd-7fe798d8c611 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb683-predictor namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 5f544c04-0540-490d-a977-518ce47ce40f kind: Event lastTimestamp: "2026-07-27T21:26:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:39Z" name: success-200-isvc-cb683-predictor.18c642a9da4cf978 namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: 0dfe0861-6677-45da-b057-aa9497b43880 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb683 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: 6e279106-dc67-4f58-a808-8ee8e2e218d9 kind: Event lastTimestamp: "2026-07-27T21:26:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:27Z" name: success-200-isvc-cb683.18c6429bde8bbbfb namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 0ec174ad-90bd-4877-8bc9-0925635af401 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb683 namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 6e279106-dc67-4f58-a808-8ee8e2e218d9 kind: Event lastTimestamp: "2026-07-27T21:26:27Z" message: InferenceService [success-200-isvc-cb683] is Ready metadata: creationTimestamp: "2026-07-27T21:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:27Z" name: success-200-isvc-cb683.18c642a7175164e7 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: f01d3679-b6a9-45f9-bc58-527b37acc2b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:16:54.535301Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d13b2-predictor-86c6db6b55-mf54z to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:16:54Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c64221a3045cf9 namespace: kserve-ci-e2e-test resourceVersion: "16815" uid: 79664ebc-b136-4d75-8c83-b718e35be0bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:16:54Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c64221bc98273e namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: bbf8f966-8239-4fef-8e8f-377377b3e826 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c64221be611f24 namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: f890cb81-fc63-44d9-a2cd-1f11ee742c67 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c64221c15fa86a namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: 9f2b8e3c-5f8f-42a9-b42f-ba95950f3add reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c64221c1c6a587 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: a0a508cf-01cc-4aab-b793-df2201ca434a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c64221c1d14a04 namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 7723431e-1591-4f13-b3ba-6f01593c511e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c64221c52df240 namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: 6e8e7141-0817-4b2f-b643-70249b8691fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:16:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:55Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c64221c591aca3 namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: 6c5ed614-3b0a-4f7e-b28b-41380eebb221 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-27T21:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:26:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:11Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c64221d5ce7ca4 namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: b8addfa1-1607-4821-9115-f1d1acbf5b13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:09Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c642a2cace9e01 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 394469f1-fba9-4ce1-a685-332c1fedb5de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:26:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:26:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:09Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c642a2cad49ced namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: c80bcf8f-51f7-4c14-a01a-baef80be2c2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: cb543262-741d-481b-b338-bb7d49c7774c kind: Event lastTimestamp: "2026-07-27T21:26:11Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:11Z" name: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z.18c642a34a432e84 namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: 4d3d094f-7b11-4611-b8b6-2d4bc48a205f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d13b2-predictor-86c6db6b55 namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: 3fed2e72-51f1-43c9-a1c4-cc6187bb4152 kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: 'Created pod: success-200-isvc-d13b2-predictor-86c6db6b55-mf54z' metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:54Z" name: success-200-isvc-d13b2-predictor-86c6db6b55.18c64221a26d48c9 namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: e467de7c-3e83-41e5-b1cc-673655995a93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: 4eb50e7c-585d-43f1-8769-1f82a2311ec3 kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: Scaled up replica set success-200-isvc-d13b2-predictor-86c6db6b55 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:54Z" name: success-200-isvc-d13b2-predictor.18c642219fae8872 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: e1056e05-e12b-43b0-9c27-7e857f5f224a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:17:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: 1010671d-b30a-432e-8d0e-82053453b9dc kind: Event lastTimestamp: "2026-07-27T21:17:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:39Z" name: success-200-isvc-d13b2-predictor.18c642251e226bf6 namespace: kserve-ci-e2e-test resourceVersion: "17325" uid: 3956a84e-a1a3-4d69-a0bb-c79dfc8169b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:17:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: 1010671d-b30a-432e-8d0e-82053453b9dc kind: Event lastTimestamp: "2026-07-27T21:17:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:39Z" name: success-200-isvc-d13b2-predictor.18c642251e82614c namespace: kserve-ci-e2e-test resourceVersion: "17326" uid: 26b541fe-9900-4fd6-ab9d-9158d3f30329 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:17:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17034" uid: 1010671d-b30a-432e-8d0e-82053453b9dc kind: Event lastTimestamp: "2026-07-27T21:18:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:18:24Z" name: success-200-isvc-d13b2-predictor.18c642289e58906a namespace: kserve-ci-e2e-test resourceVersion: "17726" uid: 30983881-eaaa-4759-b8e7-d8a8d10fc00d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:17:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d13b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17034" uid: 1010671d-b30a-432e-8d0e-82053453b9dc kind: Event lastTimestamp: "2026-07-27T21:18:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:18:24Z" name: success-200-isvc-d13b2-predictor.18c642289ed3a5f6 namespace: kserve-ci-e2e-test resourceVersion: "17727" uid: b63b80e2-8b16-4f8c-90bd-3a8e4aaa2ec0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 7adac5d0-05e9-4ad4-aa6a-d5ce0f11a2c9 kind: Event lastTimestamp: "2026-07-27T21:17:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:41Z" name: success-200-isvc-d13b2.18c642219fad18f7 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: 8329570b-4827-4bf5-a0fe-3e72c4b6355c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d13b2 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 7adac5d0-05e9-4ad4-aa6a-d5ce0f11a2c9 kind: Event lastTimestamp: "2026-07-27T21:17:41Z" message: InferenceService [success-200-isvc-d13b2] is Ready metadata: creationTimestamp: "2026-07-27T21:17:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:41Z" name: success-200-isvc-d13b2.18c6422c90bccf7d namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: 25645cfe-8f16-4c63-9086-f79214f6ff53 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:16:24.631305Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421aac999b64 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 538905ce-bec9-41a2-a0e3-90941c426e86 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d355e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421ababc1b18 namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: 39924c4b-58b4-46c9-842f-c73e774fac6f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16352" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:16:25Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421ae9e283fd namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: b0ba3de1-1ed4-4c9e-be15-a0c0ce67ba0f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421aeb927be2 namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: a19bc659-3bec-4a90-bcd7-4aa6a5db662c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421aee371d4a namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 3914415b-e10a-454c-a0c3-20069b4e6941 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421aeebadb03 namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: c429084b-0f88-4c23-8a71-f838c9bdeb10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421aeec5321c namespace: kserve-ci-e2e-test resourceVersion: "16431" uid: 45bedce3-c606-4e7f-bb2c-71e308088f35 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421af1b6432c namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: be458562-9737-48c7-aa28-d2b9c48f44c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:16:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:25Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421af2158240 namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: 150db0e4-8d64-4d16-9b4c-24c9ac8b2022 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:17:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:03Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6421b4837319d namespace: kserve-ci-e2e-test resourceVersion: "16987" uid: 74dd6457-3a21-4f8c-b5df-8dfbb1514730 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:25:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:39Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6429bd9fd60f8 namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: dc3f0da6-a198-434f-96e9-5b2b37e9d265 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: bc456f43-7c87-4e48-9c2a-118520ab8903 kind: Event lastTimestamp: "2026-07-27T21:25:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:39Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n.18c6429bd9fea08e namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 528bcb6f-c41d-4eb0-bdeb-6a1a561d1bd2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d355e-predictor-c7c6d9d4c namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: ccb60168-75f7-43a9-a3bf-af3a601960fe kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'Created pod: success-200-isvc-d355e-predictor-c7c6d9d4c-s9x8n' metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-d355e-predictor-c7c6d9d4c.18c6421aac006792 namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: 2e262fb2-0f56-4c95-b762-80904beac2df reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: a80444bd-792c-4cdd-b0af-f7e8f7c9656a kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: Scaled up replica set success-200-isvc-d355e-predictor-c7c6d9d4c from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-d355e-predictor.18c6421aa8ab3432 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: ff7c0ae6-d9a3-4b65-8e3b-d678f4b80ee6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 6f5ae117-7078-4bf1-b65e-d6ee29ee036a kind: Event lastTimestamp: "2026-07-27T21:17:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:09Z" name: success-200-isvc-d355e-predictor.18c6421e27113828 namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: 891ec0a3-258c-4cbf-9dfe-06ce3c5b33f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 6f5ae117-7078-4bf1-b65e-d6ee29ee036a kind: Event lastTimestamp: "2026-07-27T21:17:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:09Z" name: success-200-isvc-d355e-predictor.18c6421e27776254 namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: 79401a6e-4714-4e9e-8590-5ae6371070b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: 6f5ae117-7078-4bf1-b65e-d6ee29ee036a kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:54Z" name: success-200-isvc-d355e-predictor.18c64221a697b986 namespace: kserve-ci-e2e-test resourceVersion: "17487" uid: f4189bd7-2072-4075-96e4-7470996fc385 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-27T21:16:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d355e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: 6f5ae117-7078-4bf1-b65e-d6ee29ee036a kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:54Z" name: success-200-isvc-d355e-predictor.18c64221a72bf6dc namespace: kserve-ci-e2e-test resourceVersion: "17488" uid: 270bd7e0-70d1-447f-ab1f-dd383a0966b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d355e namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: cab46a98-f095-42ad-b289-936f91406f0e kind: Event lastTimestamp: "2026-07-27T21:17:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:13Z" name: success-200-isvc-d355e.18c6421aa8a65494 namespace: kserve-ci-e2e-test resourceVersion: "17086" uid: 3f0bc1f9-0ce8-42e6-9a84-9672d1633a20 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d355e namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: cab46a98-f095-42ad-b289-936f91406f0e kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'Failed to update status for InferenceService "success-200-isvc-d355e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d355e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-d355e.18c6421aa96d8ef6 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 5440b8c6-1404-4c6a-bc97-d22fba0bccea reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d355e namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: cab46a98-f095-42ad-b289-936f91406f0e kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d355e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:24Z" name: success-200-isvc-d355e.18c6421aa96de070 namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 7a7d9459-ca9f-4876-a41a-bd37e7145e3e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d355e namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: cab46a98-f095-42ad-b289-936f91406f0e kind: Event lastTimestamp: "2026-07-27T21:17:13Z" message: InferenceService [success-200-isvc-d355e] is Ready metadata: creationTimestamp: "2026-07-27T21:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:13Z" name: success-200-isvc-d355e.18c6422603981e9a namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: a7d1fd29-12ed-46f2-87fb-d222d98f3ee8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:26:49.869008Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e26c5-predictor-d5cd578f-9wk7v to ip-10-0-135-121.ec2.internal metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:26:49Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c642ac3fabe8ba namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: d70d8b71-71b7-408f-a976-8a066607263f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:26:50Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:26:50Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c642ac59602bbb namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: 66729a4b-39d6-4973-b302-3e00133a4c8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:26:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-27T21:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:50Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c642ac5ace9baf namespace: kserve-ci-e2e-test resourceVersion: "21721" uid: b12ad593-4a63-463d-b763-5ce35aa13173 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:26:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-27T21:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:50Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c642ac5e104254 namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 4c0a2770-1232-437b-b771-ab72e9463fad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:26:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-27T21:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:50Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c642ac5e8d38e9 namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 19becdc7-8891-46e0-94a2-d4c01d22651d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:26:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-27T21:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:50Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c642ac5e97efc8 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 1081a2c9-64e8-4780-b787-0228bdb51781 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:26:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-27T21:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:50Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c642ac617a31e9 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 8cb999da-b9bf-4ce9-b468-4f20345876f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:26:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:50Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c642ac61e74219 namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: c64a1f1a-e9eb-47ae-b193-03c6d3a1154f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-27T21:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:36:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:06Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c642ac7eeea56e namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 0d31191b-98e9-4bfd-9d27-9dd0845e7ceb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:04Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c6432d6aeed7b0 namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: b175e5f8-8ba2-4d49-bea8-65569f47e012 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:36:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-27T21:36:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:04Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c6432d6aef7d6c namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: 9f6d2204-a59d-467c-9976-741224f579e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:36:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 30f6f403-0fce-4bdf-9483-b6461d287931 kind: Event lastTimestamp: "2026-07-27T21:36:06Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-27T21:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:36:06Z" name: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v.18c6432df357edbc namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: 17880583-ea3e-4759-a7bc-2988f2fcec58 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-121.ec2.internal source: component: kubelet host: ip-10-0-135-121.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e26c5-predictor-d5cd578f namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: c63dfb90-d72f-4b8b-a7d8-8d1bd615134f kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: 'Created pod: success-200-isvc-e26c5-predictor-d5cd578f-9wk7v' metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:49Z" name: success-200-isvc-e26c5-predictor-d5cd578f.18c642ac3f4aaf53 namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: 38cbdbf4-8691-4261-a624-83055f38eb5f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21644" uid: 3167148f-80ee-408e-b529-2fb01b06345e kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: Scaled up replica set success-200-isvc-e26c5-predictor-d5cd578f from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:26:49Z" name: success-200-isvc-e26c5-predictor.18c642ac3d5d91fd namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 39b62dae-dd8e-44f8-ba34-4bdbdf44a537 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:27:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: 70e3c1f0-505c-46d5-89f0-87558c91d9a7 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:19Z" name: success-200-isvc-e26c5-predictor.18c642afbbac7b8f namespace: kserve-ci-e2e-test resourceVersion: "22152" uid: dac7bbdc-5afc-48e3-934d-5b851dc044a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:27:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: 70e3c1f0-505c-46d5-89f0-87558c91d9a7 kind: Event lastTimestamp: "2026-07-27T21:27:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:27:19Z" name: success-200-isvc-e26c5-predictor.18c642afbc0f8ae5 namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: bbc4f986-8d33-4b4c-b44e-68e590fc447c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: 70e3c1f0-505c-46d5-89f0-87558c91d9a7 kind: Event lastTimestamp: "2026-07-27T21:28:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:19Z" name: success-200-isvc-e26c5-predictor.18c642b6b983fef1 namespace: kserve-ci-e2e-test resourceVersion: "22758" uid: bb6617cd-3e1d-49b4-bb4b-0e1f01cdcaa9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-27T21:27:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e26c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: 70e3c1f0-505c-46d5-89f0-87558c91d9a7 kind: Event lastTimestamp: "2026-07-27T21:28:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:28:19Z" name: success-200-isvc-e26c5-predictor.18c642b6b9e8dd39 namespace: kserve-ci-e2e-test resourceVersion: "22759" uid: 7eaf7780-cf45-417c-808c-49f946ee76c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 9dcba701-23c1-4306-b2a5-5d970e46c4b6 kind: Event lastTimestamp: "2026-07-27T21:27:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:27:37Z" name: success-200-isvc-e26c5.18c642ac3d99e52c namespace: kserve-ci-e2e-test resourceVersion: "22336" uid: fee9ec98-a352-4d6c-b696-3cfbf23a97f7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 9dcba701-23c1-4306-b2a5-5d970e46c4b6 kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: 'Failed to update status for InferenceService "success-200-isvc-e26c5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e26c5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:49Z" name: success-200-isvc-e26c5.18c642ac3e1c024b namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: 8ea69600-37bd-4e5e-a816-c9fe8ba80dd2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:26:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 9dcba701-23c1-4306-b2a5-5d970e46c4b6 kind: Event lastTimestamp: "2026-07-27T21:26:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e26c5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-27T21:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:26:49Z" name: success-200-isvc-e26c5.18c642ac3e1c597a namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: 35bd3b50-8319-40b6-a4c4-13bb30cb8d1e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:27:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e26c5 namespace: kserve-ci-e2e-test resourceVersion: "22334" uid: 9dcba701-23c1-4306-b2a5-5d970e46c4b6 kind: Event lastTimestamp: "2026-07-27T21:27:37Z" message: InferenceService [success-200-isvc-e26c5] is Ready metadata: creationTimestamp: "2026-07-27T21:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:27:37Z" name: success-200-isvc-e26c5.18c642b73a4156af namespace: kserve-ci-e2e-test resourceVersion: "22335" uid: d1c10459-9226-49e9-a1e6-4d466ac09f26 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:16:14.250487Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-070ff-5949b59c9c-h4596 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 0b0c3c35-819a-45b8-beab-4cb0fffeafe6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-070ff-5949b59c9c-h4596 to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:16:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:16:14Z" name: switch-graph-070ff-5949b59c9c-h4596.18c6421841dae73d namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: de3fbc57-060e-4963-a32e-cf1f8cd80e57 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:14Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-070ff-5949b59c9c-h4596 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 0b0c3c35-819a-45b8-beab-4cb0fffeafe6 kind: Event lastTimestamp: "2026-07-27T21:16:14Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:16:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:16:14Z" name: switch-graph-070ff-5949b59c9c-h4596.18c642185af003b6 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: e4438259-9035-490a-b5a2-84d5a827ac21 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-070ff} kind: Pod name: switch-graph-070ff-5949b59c9c-h4596 namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 0b0c3c35-819a-45b8-beab-4cb0fffeafe6 kind: Event lastTimestamp: "2026-07-27T21:16:14Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" metadata: creationTimestamp: "2026-07-27T21:16:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:14Z" name: switch-graph-070ff-5949b59c9c-h4596.18c642185c2765f8 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 1468ae49-51b3-4fc4-b13c-9dcc35fb0c11 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-070ff} kind: Pod name: switch-graph-070ff-5949b59c9c-h4596 namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 0b0c3c35-819a-45b8-beab-4cb0fffeafe6 kind: Event lastTimestamp: "2026-07-27T21:16:17Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" in 2.512s (2.512s including waiting). Image size: 218232445 bytes.' metadata: creationTimestamp: "2026-07-27T21:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:17Z" name: switch-graph-070ff-5949b59c9c-h4596.18c64218f1e41a3c namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 860212dc-0989-4fca-9487-e7f3173c35b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-070ff} kind: Pod name: switch-graph-070ff-5949b59c9c-h4596 namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 0b0c3c35-819a-45b8-beab-4cb0fffeafe6 kind: Event lastTimestamp: "2026-07-27T21:16:17Z" message: 'Created container: switch-graph-070ff' metadata: creationTimestamp: "2026-07-27T21:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:17Z" name: switch-graph-070ff-5949b59c9c-h4596.18c64218f4c1c740 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: a30d7bd7-0027-4df5-99fd-e422c149ecd1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-070ff} kind: Pod name: switch-graph-070ff-5949b59c9c-h4596 namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 0b0c3c35-819a-45b8-beab-4cb0fffeafe6 kind: Event lastTimestamp: "2026-07-27T21:16:17Z" message: Started container switch-graph-070ff metadata: creationTimestamp: "2026-07-27T21:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:17Z" name: switch-graph-070ff-5949b59c9c-h4596.18c64218f533d7dc namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: b5996c50-1223-4d34-8308-260cf53da89d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-070ff} kind: Pod name: switch-graph-070ff-5949b59c9c-h4596 namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 0b0c3c35-819a-45b8-beab-4cb0fffeafe6 kind: Event lastTimestamp: "2026-07-27T21:16:24Z" message: Stopping container switch-graph-070ff metadata: creationTimestamp: "2026-07-27T21:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:24Z" name: switch-graph-070ff-5949b59c9c-h4596.18c6421aa0148a57 namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: e6c2ab3f-f66a-46f2-b5ff-6dd104bdbc79 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-070ff} kind: Pod name: switch-graph-070ff-5949b59c9c-h4596 namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 0b0c3c35-819a-45b8-beab-4cb0fffeafe6 kind: Event lastTimestamp: "2026-07-27T21:16:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:16:54Z" name: switch-graph-070ff-5949b59c9c-h4596.18c6421bbc1a5b49 namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: 08b4c2d1-1e5d-40b8-aa18-48a6da93cec3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-070ff-5949b59c9c namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: 5c0f37fc-1aaa-4824-99e3-dc0975eb4fef kind: Event lastTimestamp: "2026-07-27T21:16:14Z" message: 'Created pod: switch-graph-070ff-5949b59c9c-h4596' metadata: creationTimestamp: "2026-07-27T21:16:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:14Z" name: switch-graph-070ff-5949b59c9c.18c64218414a958c namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: b52b41ce-c06c-43fb-8016-249a407bfd43 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-070ff namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 9fd763cf-66c7-411b-9538-0d9037a3d2a0 kind: Event lastTimestamp: "2026-07-27T21:16:14Z" message: Scaled up replica set switch-graph-070ff-5949b59c9c from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:16:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:16:14Z" name: switch-graph-070ff.18c64218404e52f0 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 08238725-99db-46fa-a01a-92db2997fcaf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:16:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-070ff namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: fb63b34f-9b79-4329-9400-cda65e058f48 kind: Event lastTimestamp: "2026-07-27T21:16:14Z" message: InferenceGraph [switch-graph-070ff] is Ready metadata: creationTimestamp: "2026-07-27T21:16:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:16:14Z" name: switch-graph-070ff.18c6421844211785 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: c1bf419d-facd-40c3-b00d-75178e3ccaf8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:37:34.520039Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-beb6a-78db6d8859-vxrps namespace: kserve-ci-e2e-test resourceVersion: "27477" uid: 42109737-e494-4909-8510-7683c70697c8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-beb6a-78db6d8859-vxrps to ip-10-0-138-50.ec2.internal metadata: creationTimestamp: "2026-07-27T21:37:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:37:34Z" name: switch-graph-beb6a-78db6d8859-vxrps.18c6434257ddec64 namespace: kserve-ci-e2e-test resourceVersion: "27482" uid: 08fb665d-b6b0-40f0-a057-328c10e72613 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:34Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-beb6a-78db6d8859-vxrps namespace: kserve-ci-e2e-test resourceVersion: "27486" uid: 42109737-e494-4909-8510-7683c70697c8 kind: Event lastTimestamp: "2026-07-27T21:37:34Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:37:34Z" name: switch-graph-beb6a-78db6d8859-vxrps.18c64342715b2fb0 namespace: kserve-ci-e2e-test resourceVersion: "27498" uid: c956f408-966e-484c-a104-2ececc1a9097 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-beb6a} kind: Pod name: switch-graph-beb6a-78db6d8859-vxrps namespace: kserve-ci-e2e-test resourceVersion: "27479" uid: 42109737-e494-4909-8510-7683c70697c8 kind: Event lastTimestamp: "2026-07-27T21:37:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-27T21:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:34Z" name: switch-graph-beb6a-78db6d8859-vxrps.18c643427290124a namespace: kserve-ci-e2e-test resourceVersion: "27500" uid: fe47ae1d-148a-4657-a6c5-811cb2b46b7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-beb6a} kind: Pod name: switch-graph-beb6a-78db6d8859-vxrps namespace: kserve-ci-e2e-test resourceVersion: "27479" uid: 42109737-e494-4909-8510-7683c70697c8 kind: Event lastTimestamp: "2026-07-27T21:37:35Z" message: 'Created container: switch-graph-beb6a' metadata: creationTimestamp: "2026-07-27T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:35Z" name: switch-graph-beb6a-78db6d8859-vxrps.18c64342755d8f4f namespace: kserve-ci-e2e-test resourceVersion: "27502" uid: 34eb801c-5c78-4a21-9ba4-b6dee360af93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-beb6a} kind: Pod name: switch-graph-beb6a-78db6d8859-vxrps namespace: kserve-ci-e2e-test resourceVersion: "27479" uid: 42109737-e494-4909-8510-7683c70697c8 kind: Event lastTimestamp: "2026-07-27T21:37:35Z" message: Started container switch-graph-beb6a metadata: creationTimestamp: "2026-07-27T21:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:37:35Z" name: switch-graph-beb6a-78db6d8859-vxrps.18c6434275d25389 namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: bb8ce280-7c3d-4511-b1b7-7c4c8b1cdad1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-beb6a} kind: Pod name: switch-graph-beb6a-78db6d8859-vxrps namespace: kserve-ci-e2e-test resourceVersion: "27479" uid: 42109737-e494-4909-8510-7683c70697c8 kind: Event lastTimestamp: "2026-07-27T21:53:53Z" message: Stopping container switch-graph-beb6a metadata: creationTimestamp: "2026-07-27T21:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:53:53Z" name: switch-graph-beb6a-78db6d8859-vxrps.18c644264baf8b9d namespace: kserve-ci-e2e-test resourceVersion: "34387" uid: 795bb3fc-3d83-4b5b-9ac2-2f69c2c6e51d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-27T21:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-beb6a} kind: Pod name: switch-graph-beb6a-78db6d8859-vxrps namespace: kserve-ci-e2e-test resourceVersion: "27479" uid: 42109737-e494-4909-8510-7683c70697c8 kind: Event lastTimestamp: "2026-07-27T21:54:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:54:16Z" name: switch-graph-beb6a-78db6d8859-vxrps.18c64426ff3000e6 namespace: kserve-ci-e2e-test resourceVersion: "34630" uid: e60897b9-d706-4dc4-8991-dcb483792141 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-50.ec2.internal source: component: kubelet host: ip-10-0-138-50.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-beb6a-78db6d8859 namespace: kserve-ci-e2e-test resourceVersion: "27474" uid: c0a5f80d-c2e4-4923-80ca-4ac464b618a7 kind: Event lastTimestamp: "2026-07-27T21:37:34Z" message: 'Created pod: switch-graph-beb6a-78db6d8859-vxrps' metadata: creationTimestamp: "2026-07-27T21:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:34Z" name: switch-graph-beb6a-78db6d8859.18c64342576fdd79 namespace: kserve-ci-e2e-test resourceVersion: "27478" uid: 2520a14b-f800-4754-945e-27da8e7db955 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-beb6a namespace: kserve-ci-e2e-test resourceVersion: "27470" uid: 9b5647f2-126d-4efc-9b35-11061410d907 kind: Event lastTimestamp: "2026-07-27T21:37:34Z" message: Scaled up replica set switch-graph-beb6a-78db6d8859 from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:37:34Z" name: switch-graph-beb6a.18c6434256c3bf74 namespace: kserve-ci-e2e-test resourceVersion: "27475" uid: 6e264466-e769-40da-8056-f779d3aaf312 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-beb6a namespace: kserve-ci-e2e-test resourceVersion: "27491" uid: 6441c55e-6163-4673-8d01-b345b0d4ab59 kind: Event lastTimestamp: "2026-07-27T21:37:34Z" message: InferenceGraph [switch-graph-beb6a] is Ready metadata: creationTimestamp: "2026-07-27T21:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:37:34Z" name: switch-graph-beb6a.18c643425c26e594 namespace: kserve-ci-e2e-test resourceVersion: "27493" uid: 0662bfa4-858f-444e-9c2c-ac4684d13d1e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-beb6a namespace: kserve-ci-e2e-test resourceVersion: "27473" uid: f1478b85-a2d6-4d24-ac60-1a2f00f08ee7 kind: Event lastTimestamp: "2026-07-27T21:38:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:04Z" name: switch-graph-beb6a.18c64345d553dd8b namespace: kserve-ci-e2e-test resourceVersion: "27761" uid: 95bec898-a4ed-4fac-8c00-31ac2613c5d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-27T21:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-beb6a namespace: kserve-ci-e2e-test resourceVersion: "27473" uid: f1478b85-a2d6-4d24-ac60-1a2f00f08ee7 kind: Event lastTimestamp: "2026-07-27T21:38:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:38:04Z" name: switch-graph-beb6a.18c64345d591527f namespace: kserve-ci-e2e-test resourceVersion: "27762" uid: 87adae75-b598-43a1-933a-2e8e2e772985 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-27T21:17:24.724301Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d355e-8455db45bc-6rwmn namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: daf8ac4e-3698-4f27-a5b3-3cb8247d05c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d355e-8455db45bc-6rwmn to ip-10-0-128-201.ec2.internal metadata: creationTimestamp: "2026-07-27T21:17:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-27T21:17:24Z" name: switch-graph-d355e-8455db45bc-6rwmn.18c64228aa6c047b namespace: kserve-ci-e2e-test resourceVersion: "17197" uid: e88c4075-7244-4bd7-8278-bff82a4a55cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f49bb85bc-qqdkx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:24Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d355e-8455db45bc-6rwmn namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: daf8ac4e-3698-4f27-a5b3-3cb8247d05c9 kind: Event lastTimestamp: "2026-07-27T21:17:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d355e-serving-cert" not found' metadata: creationTimestamp: "2026-07-27T21:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:24Z" name: switch-graph-d355e-8455db45bc-6rwmn.18c64228b2ea72e7 namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 09eef4ef-d777-4c52-b12f-ac6ca80367f3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:25Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d355e-8455db45bc-6rwmn namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: daf8ac4e-3698-4f27-a5b3-3cb8247d05c9 kind: Event lastTimestamp: "2026-07-27T21:17:25Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-27T21:17:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-27T21:17:25Z" name: switch-graph-d355e-8455db45bc-6rwmn.18c64228e827f745 namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: a0c4f91c-41ee-4a7f-940b-d01bf26c9b11 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d355e} kind: Pod name: switch-graph-d355e-8455db45bc-6rwmn namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: daf8ac4e-3698-4f27-a5b3-3cb8247d05c9 kind: Event lastTimestamp: "2026-07-27T21:17:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-27T21:17:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:25Z" name: switch-graph-d355e-8455db45bc-6rwmn.18c64228e974118a namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: eff7068f-eb44-4dca-a30d-1f68cbe3bcf1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d355e} kind: Pod name: switch-graph-d355e-8455db45bc-6rwmn namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: daf8ac4e-3698-4f27-a5b3-3cb8247d05c9 kind: Event lastTimestamp: "2026-07-27T21:17:25Z" message: 'Created container: switch-graph-d355e' metadata: creationTimestamp: "2026-07-27T21:17:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:25Z" name: switch-graph-d355e-8455db45bc-6rwmn.18c64228ec664e49 namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 0bf67f3c-d148-4ac1-965e-08e716cf0537 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d355e} kind: Pod name: switch-graph-d355e-8455db45bc-6rwmn namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: daf8ac4e-3698-4f27-a5b3-3cb8247d05c9 kind: Event lastTimestamp: "2026-07-27T21:17:25Z" message: Started container switch-graph-d355e metadata: creationTimestamp: "2026-07-27T21:17:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:17:25Z" name: switch-graph-d355e-8455db45bc-6rwmn.18c64228ecdc7410 namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: a47ccd75-f67f-4d78-8ca9-1600946b71ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:25:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d355e} kind: Pod name: switch-graph-d355e-8455db45bc-6rwmn namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: daf8ac4e-3698-4f27-a5b3-3cb8247d05c9 kind: Event lastTimestamp: "2026-07-27T21:25:39Z" message: Stopping container switch-graph-d355e metadata: creationTimestamp: "2026-07-27T21:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:25:39Z" name: switch-graph-d355e-8455db45bc-6rwmn.18c6429bd35fa6d9 namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: b94b5989-34a7-406a-b7ae-1243e17382c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-27T21:25:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d355e} kind: Pod name: switch-graph-d355e-8455db45bc-6rwmn namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: daf8ac4e-3698-4f27-a5b3-3cb8247d05c9 kind: Event lastTimestamp: "2026-07-27T21:26:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-27T21:25:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-27T21:26:07Z" name: switch-graph-d355e-8455db45bc-6rwmn.18c6429c7843d43f namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: dd0a64a2-2ab1-459c-bff5-e85ab3ecd121 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-201.ec2.internal source: component: kubelet host: ip-10-0-128-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d355e-8455db45bc namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: c4570719-603b-45b0-aea3-b790577310bc kind: Event lastTimestamp: "2026-07-27T21:17:24Z" message: 'Created pod: switch-graph-d355e-8455db45bc-6rwmn' metadata: creationTimestamp: "2026-07-27T21:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:24Z" name: switch-graph-d355e-8455db45bc.18c64228a9f69c7d namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: 65cddab3-1587-432b-92b7-353d8fc9d505 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d355e namespace: kserve-ci-e2e-test resourceVersion: "17183" uid: 8dffc28f-1626-457a-b151-ebb0533ffe99 kind: Event lastTimestamp: "2026-07-27T21:17:24Z" message: Scaled up replica set switch-graph-d355e-8455db45bc from 0 to 1 metadata: creationTimestamp: "2026-07-27T21:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:24Z" name: switch-graph-d355e.18c64228a8dc5501 namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 5a6bd968-ce5e-4a23-8abc-31bbed69bf6d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d355e namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 5c81fb58-e5b9-4b55-89bf-444565b32623 kind: Event lastTimestamp: "2026-07-27T21:17:24Z" message: InferenceGraph [switch-graph-d355e] is Ready metadata: creationTimestamp: "2026-07-27T21:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-27T21:17:24Z" name: switch-graph-d355e.18c64228ae2daf8c namespace: kserve-ci-e2e-test resourceVersion: "17206" uid: 6d5f3a01-18d6-4bc8-8770-886a81f9819d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d355e namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: de558f8a-54b4-4302-b5b9-191e11e92f9b kind: Event lastTimestamp: "2026-07-27T21:17:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:39Z" name: switch-graph-d355e.18c6422c26e99282 namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: b369749e-55f8-4d72-bc0b-1d862fc6108b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d355e namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: de558f8a-54b4-4302-b5b9-191e11e92f9b kind: Event lastTimestamp: "2026-07-27T21:17:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-27T21:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:39Z" name: switch-graph-d355e.18c6422c27353099 namespace: kserve-ci-e2e-test resourceVersion: "17335" uid: 97719da9-7886-4d59-9ed1-7ccc362c17d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d355e namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: de558f8a-54b4-4302-b5b9-191e11e92f9b kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:54Z" name: switch-graph-d355e.18c6422fa5d089ce namespace: kserve-ci-e2e-test resourceVersion: "17492" uid: 32e92e53-d61d-413d-bc7f-6a1ad6083a5a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-27T21:17:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d355e namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: de558f8a-54b4-4302-b5b9-191e11e92f9b kind: Event lastTimestamp: "2026-07-27T21:17:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-27T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-27T21:17:54Z" name: switch-graph-d355e.18c6422fa6299023 namespace: kserve-ci-e2e-test resourceVersion: "17494" uid: ed7c4ce0-da66-4c9f-8aa2-ae65849bac15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34642"