--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:34:56.099845Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-03875-56c5b76d48-sbw5h namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 54488bfb-5bd8-4cf2-af48-f23398ddda74 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-03875-56c5b76d48-sbw5h to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:34:56Z" name: ensemble-graph-03875-56c5b76d48-sbw5h.18c00896c61141f2 namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: bb290682-d4d8-4055-b616-131030231676 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:56Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-03875-56c5b76d48-sbw5h namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: 54488bfb-5bd8-4cf2-af48-f23398ddda74 kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:34:56Z" name: ensemble-graph-03875-56c5b76d48-sbw5h.18c00896dffc7df6 namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: a7db49e0-d72b-48c5-82fd-907c3c4ea749 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-03875} kind: Pod name: ensemble-graph-03875-56c5b76d48-sbw5h namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: 54488bfb-5bd8-4cf2-af48-f23398ddda74 kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:56Z" name: ensemble-graph-03875-56c5b76d48-sbw5h.18c00896e187c4c2 namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: 55c6ee2c-1556-41f9-a252-588a54272f72 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-03875} kind: Pod name: ensemble-graph-03875-56c5b76d48-sbw5h namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: 54488bfb-5bd8-4cf2-af48-f23398ddda74 kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: 'Created container: ensemble-graph-03875' metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:56Z" name: ensemble-graph-03875-56c5b76d48-sbw5h.18c00896e4d37f0b namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: 03000aa3-23e9-436b-954d-dd4057c22342 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-03875} kind: Pod name: ensemble-graph-03875-56c5b76d48-sbw5h namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: 54488bfb-5bd8-4cf2-af48-f23398ddda74 kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: Started container ensemble-graph-03875 metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:56Z" name: ensemble-graph-03875-56c5b76d48-sbw5h.18c00896e590d25f namespace: kserve-ci-e2e-test resourceVersion: "21768" uid: dc7f95ee-662e-44c4-9e86-eedcab4d2a21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-03875} kind: Pod name: ensemble-graph-03875-56c5b76d48-sbw5h namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: 54488bfb-5bd8-4cf2-af48-f23398ddda74 kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: Stopping container ensemble-graph-03875 metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:06Z" name: ensemble-graph-03875-56c5b76d48-sbw5h.18c008991e9363ed namespace: kserve-ci-e2e-test resourceVersion: "21863" uid: cc4aa9cd-7c49-4308-a42d-286c03c2501b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:35:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-03875} kind: Pod name: ensemble-graph-03875-56c5b76d48-sbw5h namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: 54488bfb-5bd8-4cf2-af48-f23398ddda74 kind: Event lastTimestamp: "2026-07-07T14:35:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:33Z" name: ensemble-graph-03875-56c5b76d48-sbw5h.18c0089990ebcc4e namespace: kserve-ci-e2e-test resourceVersion: "22217" uid: b39008af-ba0b-4ac0-9982-7bae4df0e36d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-03875-56c5b76d48 namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 99bd0f56-719a-457b-8fd3-5673c2fe5ecb kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: 'Created pod: ensemble-graph-03875-56c5b76d48-sbw5h' metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:56Z" name: ensemble-graph-03875-56c5b76d48.18c00896c591330a namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 231d6b12-4d21-4709-90fc-0dcd3a2cd765 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-03875 namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: f77bb959-3cfe-44ec-b9b2-2adc1f7ad295 kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: Scaled up replica set ensemble-graph-03875-56c5b76d48 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:56Z" name: ensemble-graph-03875.18c00896c4cac033 namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 4551df03-7768-4aad-b306-f6c2e6c76dad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-03875 namespace: kserve-ci-e2e-test resourceVersion: "21754" uid: 2591dc46-dae4-4e4d-8727-557215ee7729 kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: InferenceGraph [ensemble-graph-03875] is Ready metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:34:56Z" name: ensemble-graph-03875.18c00896cadf1363 namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: 25cb78a3-c74b-4933-be15-a99ee71be693 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:36:06.381375Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm namespace: kserve-ci-e2e-test resourceVersion: "22702" uid: 10971237-a1b1-46e5-bbd2-24d949011576 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-fe15b-7ffc6fdfb6-xk2pm to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:36:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:36:06Z" name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm.18c008a7232c4cda namespace: kserve-ci-e2e-test resourceVersion: "22708" uid: 0a6d562a-ffaf-4e1e-b6bc-42592650fc60 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:06Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: 10971237-a1b1-46e5-bbd2-24d949011576 kind: Event lastTimestamp: "2026-07-07T14:36:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-fe15b-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:06Z" name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm.18c008a7303e1335 namespace: kserve-ci-e2e-test resourceVersion: "22721" uid: a77eddd9-ea6c-48f9-b149-8592eec269fb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:07Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm namespace: kserve-ci-e2e-test resourceVersion: "22714" uid: 10971237-a1b1-46e5-bbd2-24d949011576 kind: Event lastTimestamp: "2026-07-07T14:36:07Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:36:07Z" name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm.18c008a760fa3668 namespace: kserve-ci-e2e-test resourceVersion: "22731" uid: b533cd26-088d-4c97-9616-a0421bb9297b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe15b} kind: Pod name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: 10971237-a1b1-46e5-bbd2-24d949011576 kind: Event lastTimestamp: "2026-07-07T14:36:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:07Z" name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm.18c008a76271a8a3 namespace: kserve-ci-e2e-test resourceVersion: "22733" uid: 8c34942a-6e83-47b9-a0c6-0f9ced0b4d83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe15b} kind: Pod name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: 10971237-a1b1-46e5-bbd2-24d949011576 kind: Event lastTimestamp: "2026-07-07T14:36:07Z" message: 'Created container: ensemble-graph-fe15b' metadata: creationTimestamp: "2026-07-07T14:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:07Z" name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm.18c008a766a5626e namespace: kserve-ci-e2e-test resourceVersion: "22734" uid: 31c9959c-8daa-4db0-9e9b-62cc4ba18daf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe15b} kind: Pod name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: 10971237-a1b1-46e5-bbd2-24d949011576 kind: Event lastTimestamp: "2026-07-07T14:36:07Z" message: Started container ensemble-graph-fe15b metadata: creationTimestamp: "2026-07-07T14:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:07Z" name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm.18c008a7671d434f namespace: kserve-ci-e2e-test resourceVersion: "22735" uid: 88fcfab5-5943-4319-befa-8f4a78ea383e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe15b} kind: Pod name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: 10971237-a1b1-46e5-bbd2-24d949011576 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Stopping container ensemble-graph-fe15b metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm.18c0091a523d641b namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: ddaeb134-bf5b-4c46-93b3-6b9698e60beb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:44:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe15b} kind: Pod name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: 10971237-a1b1-46e5-bbd2-24d949011576 kind: Event lastTimestamp: "2026-07-07T14:44:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:49Z" name: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm.18c0091b17037b5a namespace: kserve-ci-e2e-test resourceVersion: "26847" uid: 4809f88d-e26d-4c18-86e9-0945fa70a765 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-fe15b-7ffc6fdfb6 namespace: kserve-ci-e2e-test resourceVersion: "22699" uid: 630362cc-9818-4b10-ab47-1f644d9a43e4 kind: Event lastTimestamp: "2026-07-07T14:36:06Z" message: 'Created pod: ensemble-graph-fe15b-7ffc6fdfb6-xk2pm' metadata: creationTimestamp: "2026-07-07T14:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:06Z" name: ensemble-graph-fe15b-7ffc6fdfb6.18c008a722a6ddc6 namespace: kserve-ci-e2e-test resourceVersion: "22704" uid: 048a5b58-6935-45ef-bbc6-3c4e551124a5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-fe15b namespace: kserve-ci-e2e-test resourceVersion: "22697" uid: 7e9e4ada-fc7f-44f7-8d50-9a2dfa601e2b kind: Event lastTimestamp: "2026-07-07T14:36:06Z" message: Scaled up replica set ensemble-graph-fe15b-7ffc6fdfb6 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:06Z" name: ensemble-graph-fe15b.18c008a721ce9f6b namespace: kserve-ci-e2e-test resourceVersion: "22700" uid: 6c5c42a4-a296-436d-9fb9-e80dca688fa7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-fe15b namespace: kserve-ci-e2e-test resourceVersion: "22717" uid: 9c1598a1-0db1-4d08-831c-78691ac06a52 kind: Event lastTimestamp: "2026-07-07T14:36:06Z" message: InferenceGraph [ensemble-graph-fe15b] is Ready metadata: creationTimestamp: "2026-07-07T14:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:36:06Z" name: ensemble-graph-fe15b.18c008a7271a44cc namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: 33af2065-57c6-4c49-8ed9-d6e5278ea2e2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:36:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-fe15b namespace: kserve-ci-e2e-test resourceVersion: "22698" uid: c0108f7c-dea7-4710-83c3-539cc41058be kind: Event lastTimestamp: "2026-07-07T14:36:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:36Z" name: ensemble-graph-fe15b.18c008aaa081d8d5 namespace: kserve-ci-e2e-test resourceVersion: "22973" uid: 58325c58-c353-4660-b9ea-5d8cc5771f7c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:36:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-fe15b namespace: kserve-ci-e2e-test resourceVersion: "22698" uid: c0108f7c-dea7-4710-83c3-539cc41058be kind: Event lastTimestamp: "2026-07-07T14:36:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:36Z" name: ensemble-graph-fe15b.18c008aaa0ed93ba namespace: kserve-ci-e2e-test resourceVersion: "22974" uid: a33c6fec-aa03-4c28-a18f-e505aba5cf4d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:33:56.141379Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-03875-predictor-7f99c97bbb-2kzgp to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:33:56Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c00888d043ba71 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: bcf77c3c-057b-4d54-9470-b34504a4a89f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-03875-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c00888dbc6a43e namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: 980f6abd-692b-498b-be39-b75367c8f28a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:33:57Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:33:57Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c008890d784e5b namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 0670fe93-840f-4c26-8c16-4ace511f4b88 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:33:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:57Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c008890ef337f2 namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 5e5d6791-2602-4a3b-bd5b-aa87b807b917 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:33:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:57Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c008891207187d namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: f4b4ad4f-d08c-4b3f-b7e8-d2122970ab71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:33:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:57Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c0088912707a4e namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: b34daa6e-87a9-4800-93bb-354992c1b7cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:33:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:57Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c008891278fcbd namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: ea72e6be-8b6e-4cfc-b026-53e483f64386 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:33:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:57Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c008891534311b namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: 7f2cf0df-f1a0-4146-a639-3a8e12e9db4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:33:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:57Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c0088915a07108 namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 9562c1e3-1b48-4e0a-ba0e-e586e173500e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:34:33Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:33Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c008891f5734a6 namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: 8a9a2a8b-24a1-43cb-b2b0-8b85a047f4d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:06Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c0089930043d13 namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 7d3ae0f9-2fa8-4f44-9903-74c206546ca3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:06Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c008993004c181 namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: 3fb28bea-3f98-47ca-b2f6-e89a2c8dfb59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 68ee556b-1eac-454f-a609-5dd6c6de0cc9 kind: Event lastTimestamp: "2026-07-07T14:35:08Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:08Z" name: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp.18c00899a777e8bc namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: 4c075d14-c34c-4fec-99a1-0f8f6a0da1a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-03875-predictor-7f99c97bbb namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 71ebb7d5-656d-4c86-bf70-83705218be72 kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: 'Created pod: error-404-isvc-03875-predictor-7f99c97bbb-2kzgp' metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:56Z" name: error-404-isvc-03875-predictor-7f99c97bbb.18c00888cfb57a01 namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 7db30194-7e56-46ba-be18-c404498f7b16 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: b72a3a91-6769-43fe-b21b-373e2aa3378b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Scaled up replica set error-404-isvc-03875-predictor-7f99c97bbb from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:56Z" name: error-404-isvc-03875-predictor.18c00888cdde1340 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 8592ef97-8c7f-4125-8f2f-3aab1178ab7a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:34:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 3dc05b91-9f8c-449e-82a6-ba46c335d11d kind: Event lastTimestamp: "2026-07-07T14:34:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:26Z" name: error-404-isvc-03875-predictor.18c0088c4c5d6567 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 39cc2fe0-ae2b-407f-9535-5e1e8f9f31c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:34:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 3dc05b91-9f8c-449e-82a6-ba46c335d11d kind: Event lastTimestamp: "2026-07-07T14:34:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:26Z" name: error-404-isvc-03875-predictor.18c0088c4cbf48fa namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: c63ff6ed-e5df-4157-89f4-f043a74c148d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:34:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 3dc05b91-9f8c-449e-82a6-ba46c335d11d kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:56Z" name: error-404-isvc-03875-predictor.18c008934a3f77b4 namespace: kserve-ci-e2e-test resourceVersion: "21751" uid: 14a9b00d-054b-41b5-8de3-eb19edc55430 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:34:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 3dc05b91-9f8c-449e-82a6-ba46c335d11d kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:56Z" name: error-404-isvc-03875-predictor.18c008934ab48b90 namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: fadd68e8-a7fc-419e-a6b6-ec16c7a82396 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03875 namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 43427058-c290-43ce-97c7-f7d7c6b4aa8b kind: Event lastTimestamp: "2026-07-07T14:34:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:34:43Z" name: error-404-isvc-03875.18c00888cde6ebee namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 91ac2ee4-c4fd-4a82-b6ad-ed282342009f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03875 namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 43427058-c290-43ce-97c7-f7d7c6b4aa8b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-03875": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-03875": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:56Z" name: error-404-isvc-03875.18c00888ce662f05 namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: eacdfe1b-fcfd-41e0-9034-3400dae9d2d0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03875 namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 43427058-c290-43ce-97c7-f7d7c6b4aa8b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-03875": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:56Z" name: error-404-isvc-03875.18c00888ce6683db namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 87a66453-12c8-48cf-97ff-27ba9c8da063 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03875 namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 43427058-c290-43ce-97c7-f7d7c6b4aa8b kind: Event lastTimestamp: "2026-07-07T14:34:43Z" message: InferenceService [error-404-isvc-03875] is Ready metadata: creationTimestamp: "2026-07-07T14:34:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:34:43Z" name: error-404-isvc-03875.18c00893da7788f6 namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: aea76ad2-29a0-45eb-a1b8-37dcbcb92825 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:45:11.060565Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-36f00-predictor-cb475dffd-22vxb to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:45:11Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c00925f49522e3 namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: 503f1061-1c4c-4c22-9f63-b0cbcb32d301 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T14:45:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-36f00-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:11Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c0092602ca0c62 namespace: kserve-ci-e2e-test resourceVersion: "27139" uid: f2eeb519-a290-4771-84fd-7298e325a3d5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27121" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:45:12Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c0092631d2f9b9 namespace: kserve-ci-e2e-test resourceVersion: "27155" uid: 0b608a10-16fb-41a0-bb1d-85ddcb76b00d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c00926335bfb09 namespace: kserve-ci-e2e-test resourceVersion: "27159" uid: f959789e-a7f1-4447-8952-456a5503b6cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c0092636c29589 namespace: kserve-ci-e2e-test resourceVersion: "27162" uid: 058330a1-3e51-4c66-bbf1-2532deb8737d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c00926372e7abf namespace: kserve-ci-e2e-test resourceVersion: "27163" uid: 22367149-11a5-4590-b87e-fe130dea52e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c0092637370ddb namespace: kserve-ci-e2e-test resourceVersion: "27164" uid: 312ac28b-0e3a-440a-8772-7da907a7823b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c009263a03b9e8 namespace: kserve-ci-e2e-test resourceVersion: "27165" uid: 209bb98d-4c1b-4154-8425-3225061943ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c009263a6cef31 namespace: kserve-ci-e2e-test resourceVersion: "27166" uid: cc80aa2c-31fd-409d-9ba6-69faf6e94a9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T14:45:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:48Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c009265370282c namespace: kserve-ci-e2e-test resourceVersion: "27674" uid: f8770099-4ee0-4ce5-b7dc-cc2dba75a74b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T15:02:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:02:30Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c00a17f5430e25 namespace: kserve-ci-e2e-test resourceVersion: "34795" uid: c88e651e-6b5c-4c8a-86b5-8bf65cd70c3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-36f00-predictor-cb475dffd-22vxb namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fface55e-8428-4b30-aef5-14f023cb75ae kind: Event lastTimestamp: "2026-07-07T15:02:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:02:30Z" name: error-404-isvc-36f00-predictor-cb475dffd-22vxb.18c00a17f5442d71 namespace: kserve-ci-e2e-test resourceVersion: "34796" uid: 6b5d82f1-5f3e-464f-9751-9f273d21c355 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-36f00-predictor-cb475dffd namespace: kserve-ci-e2e-test resourceVersion: "27098" uid: ee3770f7-2e7e-4cde-98fe-4047c03a1df7 kind: Event lastTimestamp: "2026-07-07T14:45:11Z" message: 'Created pod: error-404-isvc-36f00-predictor-cb475dffd-22vxb' metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:11Z" name: error-404-isvc-36f00-predictor-cb475dffd.18c00925f4283971 namespace: kserve-ci-e2e-test resourceVersion: "27106" uid: e7640e95-4731-4c48-8dd7-f781cc7cc02e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: 30d6bc7d-78e9-4eb7-8cd8-f1671ff3b7b5 kind: Event lastTimestamp: "2026-07-07T14:45:11Z" message: Scaled up replica set error-404-isvc-36f00-predictor-cb475dffd from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:11Z" name: error-404-isvc-36f00-predictor.18c00925f2a85eb0 namespace: kserve-ci-e2e-test resourceVersion: "27099" uid: 9df541a5-fe3b-4fdb-91a8-26a85c99aa4f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:45:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: eb0dc2d9-5c96-46b0-852e-55b3e43d1986 kind: Event lastTimestamp: "2026-07-07T14:45:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:56Z" name: error-404-isvc-36f00-predictor.18c009297116853b namespace: kserve-ci-e2e-test resourceVersion: "27732" uid: e4b46760-8432-4adc-8a8b-5bb80cfe4924 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:45:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: eb0dc2d9-5c96-46b0-852e-55b3e43d1986 kind: Event lastTimestamp: "2026-07-07T14:45:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:56Z" name: error-404-isvc-36f00-predictor.18c00929717b016f namespace: kserve-ci-e2e-test resourceVersion: "27733" uid: b8ae0386-fa60-4719-8207-92895ce9f2bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:45:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27339" uid: eb0dc2d9-5c96-46b0-852e-55b3e43d1986 kind: Event lastTimestamp: "2026-07-07T14:46:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:26Z" name: error-404-isvc-36f00-predictor.18c0092cf01f29cf namespace: kserve-ci-e2e-test resourceVersion: "28053" uid: f4e390c4-de37-4b69-834e-2b1eae1d72f0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:45:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27339" uid: eb0dc2d9-5c96-46b0-852e-55b3e43d1986 kind: Event lastTimestamp: "2026-07-07T14:46:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:26Z" name: error-404-isvc-36f00-predictor.18c0092cf087119f namespace: kserve-ci-e2e-test resourceVersion: "28054" uid: e5360870-7c42-4cc1-a5d3-64b76c986ca0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27081" uid: 90197b9b-5094-44b0-bb01-1671797c7129 kind: Event lastTimestamp: "2026-07-07T14:45:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:58Z" name: error-404-isvc-36f00.18c00925f2aa2e35 namespace: kserve-ci-e2e-test resourceVersion: "27763" uid: 22797b75-0db3-4970-8bb0-d1c76cbec6c6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27081" uid: 90197b9b-5094-44b0-bb01-1671797c7129 kind: Event lastTimestamp: "2026-07-07T14:45:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-36f00": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-36f00": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:11Z" name: error-404-isvc-36f00.18c00925f318e5dd namespace: kserve-ci-e2e-test resourceVersion: "27102" uid: 54da036a-e36f-4679-9642-42b414c56cc8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27081" uid: 90197b9b-5094-44b0-bb01-1671797c7129 kind: Event lastTimestamp: "2026-07-07T14:45:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-36f00": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:11Z" name: error-404-isvc-36f00.18c00925f31935d2 namespace: kserve-ci-e2e-test resourceVersion: "27103" uid: b280c9f0-f8aa-4794-8853-540e987a3571 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27759" uid: 90197b9b-5094-44b0-bb01-1671797c7129 kind: Event lastTimestamp: "2026-07-07T14:45:58Z" message: InferenceService [error-404-isvc-36f00] is Ready metadata: creationTimestamp: "2026-07-07T14:45:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:58Z" name: error-404-isvc-36f00.18c009310ee259a1 namespace: kserve-ci-e2e-test resourceVersion: "27760" uid: 06ab4843-400e-48a6-bb42-59de99b2a2f1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:24:31.116473Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-520df-predictor-696bbdd946-ns2nv to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c008054227dd3c namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: 8bd45991-51c2-4026-96b1-ee90d08489d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c008055b4a3f9e namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: f3427485-5905-47f1-a1d7-54090d34d45c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c008055cf1978b namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: c9156c7b-9362-4e63-96eb-107590652eb3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c00805601be59f namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: a2a79fbc-4a4a-4790-b825-106e7eede4f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c0080560773553 namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: a18fadeb-9465-4aff-90e8-15c32097dc9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c0080560808075 namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: 6bd77c3d-abc1-4c80-b9c1-d1462e276216 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c008056436bc73 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: bbb92dca-baf5-4fab-ab4c-5878327ccf27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c0080564920649 namespace: kserve-ci-e2e-test resourceVersion: "16642" uid: 603aff51-1f79-4c74-b817-665e26df5849 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:24:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:25:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:24:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:08Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c00805abc54525 namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 89d6fdfe-690a-498c-8d7b-f138eb705c47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c00888d3791ca9 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: f039b5af-bcaf-4647-ae3f-45c8d1ee76fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-520df-predictor-696bbdd946-ns2nv namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 3b481e4c-1994-4977-a83d-aec0e20ab4ca kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: error-404-isvc-520df-predictor-696bbdd946-ns2nv.18c00888d37aae63 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c8fefbf7-136a-4af3-a83c-d7e7b3885ca9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-520df-predictor-696bbdd946 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 4dde97bb-7b24-4d16-84b2-3fcba89f7b09 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: 'Created pod: error-404-isvc-520df-predictor-696bbdd946-ns2nv' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor-696bbdd946.18c00805416b9e3e namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: eef77668-609b-4c9f-8e80-5234005623f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: 1b92dd7b-27a8-4f14-8f72-11716d8efc4a kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Scaled up replica set error-404-isvc-520df-predictor-696bbdd946 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df-predictor.18c008053faad1f5 namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: 44c6b7bd-73b0-4480-be48-526f46903212 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:24:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 40e4ae2b-a995-426d-9f5f-9b64680e6285 kind: Event lastTimestamp: "2026-07-07T14:25:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:16Z" name: error-404-isvc-520df-predictor.18c00808be17cec7 namespace: kserve-ci-e2e-test resourceVersion: "17034" uid: 33a2ec0e-db1b-48ff-9112-96d57fd3baf1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:24:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 40e4ae2b-a995-426d-9f5f-9b64680e6285 kind: Event lastTimestamp: "2026-07-07T14:25:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:16Z" name: error-404-isvc-520df-predictor.18c00808be8cfe73 namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: a21aba6a-bf6a-4265-ab5f-fe0f4d614070 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: 40e4ae2b-a995-426d-9f5f-9b64680e6285 kind: Event lastTimestamp: "2026-07-07T14:25:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:46Z" name: error-404-isvc-520df-predictor.18c008133b273d06 namespace: kserve-ci-e2e-test resourceVersion: "17531" uid: 01e2750d-2c03-43e4-8910-b7e0eca661f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: 40e4ae2b-a995-426d-9f5f-9b64680e6285 kind: Event lastTimestamp: "2026-07-07T14:25:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:46Z" name: error-404-isvc-520df-predictor.18c008133bd5b63f namespace: kserve-ci-e2e-test resourceVersion: "17532" uid: be679df2-f239-48a1-bb98-c50bd89b2eaa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-520df namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 0f30e13f-8a92-47d0-b3d9-1aed9b2f0e85 kind: Event lastTimestamp: "2026-07-07T14:25:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:19Z" name: error-404-isvc-520df.18c008053faccf4b namespace: kserve-ci-e2e-test resourceVersion: "17072" uid: b6725212-ff3e-4d20-93ce-5ba4d6b6ce02 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-520df namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 0f30e13f-8a92-47d0-b3d9-1aed9b2f0e85 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-520df": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-520df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df.18c0080540163361 namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: 8b5c75ef-ee92-44fa-a3a5-481b8e2e983d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-520df namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 0f30e13f-8a92-47d0-b3d9-1aed9b2f0e85 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-520df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:31Z" name: error-404-isvc-520df.18c0080540168f4e namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 8f71b7a7-14c7-4eb2-90ae-df6eab8d8340 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-520df namespace: kserve-ci-e2e-test resourceVersion: "17070" uid: 0f30e13f-8a92-47d0-b3d9-1aed9b2f0e85 kind: Event lastTimestamp: "2026-07-07T14:25:19Z" message: InferenceService [error-404-isvc-520df] is Ready metadata: creationTimestamp: "2026-07-07T14:25:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:19Z" name: error-404-isvc-520df.18c0081068518c7a namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: b687db51-67c4-4f36-845c-452fa219fe47 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:44:21.358978Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0091a62232407 namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 88f7db3f-a51a-4954-a543-00e21902994d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0091a7bb055d0 namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: 14951dd4-9be5-4c16-9463-f0fc87f89afb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0091a7d12e893 namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: 5b5d73e7-08d1-4296-ad27-238f3a4cdd05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0091a7fceb68a namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 45c09e9b-0af7-4ee8-af03-19a9b77aebf5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0091a80391e3d namespace: kserve-ci-e2e-test resourceVersion: "26612" uid: f9cb1ca2-f656-4704-8fc0-2b9bfc069142 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0091a8041b464 namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 89116406-192e-40d8-8e66-744e1664e922 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0091a832e00f8 namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: 9cc3b769-c52c-4497-b1ac-78d5dbd34462 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0091a839594a3 namespace: kserve-ci-e2e-test resourceVersion: "26616" uid: 33e649a9-e17f-4a63-9032-848d0d299c7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:44:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:58Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0091aa0f603aa namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 58ee6d28-b6ec-454a-864e-292d78427dc0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:31Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0092ac1117a30 namespace: kserve-ci-e2e-test resourceVersion: "27486" uid: 712998d5-f0db-4b18-8910-da866c9dadeb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:31Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0092ac11245db namespace: kserve-ci-e2e-test resourceVersion: "27487" uid: 03344008-47b7-4842-ab9b-4829fb8b7847 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 42b9e97d-579f-4172-be60-2e8a3b22930a kind: Event lastTimestamp: "2026-07-07T14:45:33Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:33Z" name: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn.18c0092b29363f80 namespace: kserve-ci-e2e-test resourceVersion: "27529" uid: a353db66-aa4f-4267-b4b4-53fca32a3fec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6e67f-predictor-88d98bb6d namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: 0e053fa3-06ac-4c1a-b476-e9af08634e18 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: 'Created pod: error-404-isvc-6e67f-predictor-88d98bb6d-dl8kn' metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor-88d98bb6d.18c0091a61c75f0f namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: 3c1122c7-715b-4cce-af60-0b34897b1a81 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: 44681ed2-5051-493d-aadb-cf68ec068cb7 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Scaled up replica set error-404-isvc-6e67f-predictor-88d98bb6d from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f-predictor.18c0091a5ff19c9d namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: 08192bd4-b62f-4cc1-9619-a13298e0b394 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:44:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: 9aed1fe1-e1d5-49d9-8efd-b2d68d090c55 kind: Event lastTimestamp: "2026-07-07T14:44:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:44:51Z" name: error-404-isvc-6e67f-predictor.18c0091dde489479 namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: 296db986-b001-47c8-9215-ed73576d2c80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:44:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: 9aed1fe1-e1d5-49d9-8efd-b2d68d090c55 kind: Event lastTimestamp: "2026-07-07T14:44:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:44:51Z" name: error-404-isvc-6e67f-predictor.18c0091ddea9a68c namespace: kserve-ci-e2e-test resourceVersion: "26867" uid: 3e7fa7d9-b32c-4b58-86ef-efe03c7910ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:45:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: 9aed1fe1-e1d5-49d9-8efd-b2d68d090c55 kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:21Z" name: error-404-isvc-6e67f-predictor.18c00924dc28d2a8 namespace: kserve-ci-e2e-test resourceVersion: "27284" uid: 11eb9431-5ddb-4415-a3c6-34fae61cbc52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:45:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: 9aed1fe1-e1d5-49d9-8efd-b2d68d090c55 kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:21Z" name: error-404-isvc-6e67f-predictor.18c00924dcdf8bff namespace: kserve-ci-e2e-test resourceVersion: "27287" uid: 01f77bbe-3c47-4f81-910a-9bd767623e7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e67f namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: dfc76b84-9ab0-4b37-b9ed-f029d809c8c5 kind: Event lastTimestamp: "2026-07-07T14:45:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:08Z" name: error-404-isvc-6e67f.18c0091a5ff3f6b7 namespace: kserve-ci-e2e-test resourceVersion: "27000" uid: 5b2849ad-fc1f-4c90-a3ff-ed3cc299757b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e67f namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: dfc76b84-9ab0-4b37-b9ed-f029d809c8c5 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-6e67f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6e67f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f.18c0091a609166e0 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 8d92277a-8739-4859-816f-cc21cf25d264 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e67f namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: dfc76b84-9ab0-4b37-b9ed-f029d809c8c5 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6e67f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-6e67f.18c0091a6091bb2c namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: e9fcbae6-8568-44f7-bf9d-766c0400b6f6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e67f namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: dfc76b84-9ab0-4b37-b9ed-f029d809c8c5 kind: Event lastTimestamp: "2026-07-07T14:45:08Z" message: InferenceService [error-404-isvc-6e67f] is Ready metadata: creationTimestamp: "2026-07-07T14:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:08Z" name: error-404-isvc-6e67f.18c009255c822fb9 namespace: kserve-ci-e2e-test resourceVersion: "26999" uid: 88356e8f-0978-4b87-ac5c-4bc3df2bf343 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:45:31.629215Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27463" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7beb0-predictor-5754d884b4-zxsts to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:45:31Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092abe91e34c namespace: kserve-ci-e2e-test resourceVersion: "27468" uid: 734b3ba1-c84b-4195-8758-c2940fb86d8e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7beb0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:31Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092acc0b4d1e namespace: kserve-ci-e2e-test resourceVersion: "27495" uid: ff17aea0-ac9c-4bac-8114-8cc957be62f9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27484" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:45:32Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092afc0d71a0 namespace: kserve-ci-e2e-test resourceVersion: "27512" uid: a950d219-777c-4329-971c-09b5e1236670 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092afd6ad047 namespace: kserve-ci-e2e-test resourceVersion: "27517" uid: 5db962e5-b7b7-4f54-ae27-835dfdc7edc3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092b003c6072 namespace: kserve-ci-e2e-test resourceVersion: "27521" uid: 1b3878bc-72d8-44c8-80f4-24da1b1982d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092b00a9c6e2 namespace: kserve-ci-e2e-test resourceVersion: "27522" uid: dd853545-b336-438b-9e3f-7b772e732166 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092b00b26c07 namespace: kserve-ci-e2e-test resourceVersion: "27523" uid: 211d0dc6-0efe-4cff-935c-5d6d0ff6ff26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092b03ac5438 namespace: kserve-ci-e2e-test resourceVersion: "27525" uid: 862817ac-125c-4729-a744-19fb7be77411 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092b041920ad namespace: kserve-ci-e2e-test resourceVersion: "27526" uid: 87a5ded9-ce15-4e9b-9fb3-40a2cdf2055e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:46:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:09Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c0092b3d87f571 namespace: kserve-ci-e2e-test resourceVersion: "27857" uid: 4df6df35-8906-4065-86e6-f6ccdcd40dc3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:54:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:54:46Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c009abf329fa23 namespace: kserve-ci-e2e-test resourceVersion: "31590" uid: 60c5f923-77e9-423a-9c6b-a18a676722a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 4e2b4d1b-83c1-4c6e-969d-3beaaaedad89 kind: Event lastTimestamp: "2026-07-07T14:54:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:54:46Z" name: error-404-isvc-7beb0-predictor-5754d884b4-zxsts.18c009abf32aa220 namespace: kserve-ci-e2e-test resourceVersion: "31591" uid: 64bf843d-6092-4906-83f5-edd63e81ff41 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7beb0-predictor-5754d884b4 namespace: kserve-ci-e2e-test resourceVersion: "27456" uid: 9d6ca8fd-5cb3-4ded-90b4-769de0938e95 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: 'Created pod: error-404-isvc-7beb0-predictor-5754d884b4-zxsts' metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:31Z" name: error-404-isvc-7beb0-predictor-5754d884b4.18c0092abe1fb395 namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: af81910f-b7c5-4da1-9fa7-e3726a52d4c8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: ea1c5a0e-5624-4704-8a6c-3f1b9c1cf8e5 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: Scaled up replica set error-404-isvc-7beb0-predictor-5754d884b4 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:31Z" name: error-404-isvc-7beb0-predictor.18c0092abc8ae1c3 namespace: kserve-ci-e2e-test resourceVersion: "27457" uid: 64c31481-7587-438c-9f9b-d3a1cd5dc8ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27455" uid: 2523afe1-972d-4846-afb0-6ef3db5f4b57 kind: Event lastTimestamp: "2026-07-07T14:46:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:46Z" name: error-404-isvc-7beb0-predictor.18c0092e3b18af80 namespace: kserve-ci-e2e-test resourceVersion: "28246" uid: 054f04d6-f518-4309-bd3a-6b753c65cf84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27455" uid: 2523afe1-972d-4846-afb0-6ef3db5f4b57 kind: Event lastTimestamp: "2026-07-07T14:46:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:46Z" name: error-404-isvc-7beb0-predictor.18c0092e3b92efc4 namespace: kserve-ci-e2e-test resourceVersion: "28247" uid: 6772a393-6687-4af9-b02a-f75475d2f242 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27661" uid: 2523afe1-972d-4846-afb0-6ef3db5f4b57 kind: Event lastTimestamp: "2026-07-07T14:46:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:01Z" name: error-404-isvc-7beb0-predictor.18c00931baa6ec42 namespace: kserve-ci-e2e-test resourceVersion: "27797" uid: 111cb9b3-dab9-4908-8efe-e13ad0b2cc0d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27661" uid: 2523afe1-972d-4846-afb0-6ef3db5f4b57 kind: Event lastTimestamp: "2026-07-07T14:46:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:01Z" name: error-404-isvc-7beb0-predictor.18c00931bb3a5853 namespace: kserve-ci-e2e-test resourceVersion: "27798" uid: 8def1d33-2fc2-4134-a46c-3cd3785b8c0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 0868767f-1005-4f60-a450-8d9792041f5c kind: Event lastTimestamp: "2026-07-07T14:46:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:46:19Z" name: error-404-isvc-7beb0.18c0092abc8edecd namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: 095412bc-fd62-4962-b0b5-f651f1779178 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 0868767f-1005-4f60-a450-8d9792041f5c kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-7beb0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7beb0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:31Z" name: error-404-isvc-7beb0.18c0092abcee74c4 namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: fdfb579c-f588-4217-9211-abe7393c0ae9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 0868767f-1005-4f60-a450-8d9792041f5c kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7beb0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:31Z" name: error-404-isvc-7beb0.18c0092abceeb490 namespace: kserve-ci-e2e-test resourceVersion: "27462" uid: bef4b208-aab9-45a5-8f16-0c8d020d06ec reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "27992" uid: 0868767f-1005-4f60-a450-8d9792041f5c kind: Event lastTimestamp: "2026-07-07T14:46:19Z" message: InferenceService [error-404-isvc-7beb0] is Ready metadata: creationTimestamp: "2026-07-07T14:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:46:19Z" name: error-404-isvc-7beb0.18c00935f98b15a8 namespace: kserve-ci-e2e-test resourceVersion: "27993" uid: eda3f088-f957-4c64-9862-5948c0732259 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:23:01.001593Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:23:01Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f046e3e123 namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: f243464b-5ded-467e-91ad-187fd172bce1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:23:01Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f060c3dc2b namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 93e2098b-3abf-496d-9512-f82d6faac557 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f062d33528 namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 15c224db-1845-4716-8fd1-65f1789a4707 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:23:14Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" in 12.724s (12.724s including waiting). Image size: 1334625901 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:14Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f359442142 namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 15a05521-52de-4fbe-ae89-16612328f3d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:23:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:14Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f35c022ee8 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: d64cd9f0-77a6-43b0-ae45-695aed04b8d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:23:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:14Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f35c5f3120 namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 4dcf9598-5a36-4d2a-9ba4-cd0763c8193e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:23:14Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:14Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f35c6ac83a namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: d099d8a1-13d1-4b6d-a59e-9936ab88249b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:23:16Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.086s (2.086s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:16Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f3d8cd4a3d namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: 28c6fc51-60b5-4a66-b570-67489ed821bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:23:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:16Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f3dba553ef namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 5a08e6ce-b766-46f1-a9dd-8e86de3beff7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:23:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:16Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f3dc00f7b7 namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: eda32b91-9eb3-4f14-bfcd-967280e675a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T14:23:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:24:33Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:33Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c007f42024bced namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: 155b579f-e849-4168-8374-3033a6ba6499 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c008053697b20f namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: dcbf5bf4-fc1d-4741-a026-5302a9f967f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c00805369829c1 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 6f34d818-bf19-430d-b430-6eebd4bd43f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 08e9c248-50c3-4c8a-98ca-0f23ca9df0f9 kind: Event lastTimestamp: "2026-07-07T14:24:33Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:24:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:33Z" name: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7.18c00805d25103b1 namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 4a589e69-b189-4c45-8764-ad5a31b655be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a2dea-predictor-55844fbd46 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 1fd3d249-abbc-4bf5-9979-ea789e7524f2 kind: Event lastTimestamp: "2026-07-07T14:23:00Z" message: 'Created pod: error-404-isvc-a2dea-predictor-55844fbd46-hx9m7' metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:00Z" name: error-404-isvc-a2dea-predictor-55844fbd46.18c007f0464b1383 namespace: kserve-ci-e2e-test resourceVersion: "15547" uid: a480974b-574d-4a59-901f-bc6e7041fbae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: 1ada576a-6a8a-48cd-b078-4c421f46bed9 kind: Event lastTimestamp: "2026-07-07T14:23:00Z" message: Scaled up replica set error-404-isvc-a2dea-predictor-55844fbd46 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:00Z" name: error-404-isvc-a2dea-predictor.18c007f044a3e79f namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 16d5703d-3285-4e33-b51b-c3a1485bccad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:23:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 2ac8595c-2eff-4a4c-a14c-2dd908d68279 kind: Event lastTimestamp: "2026-07-07T14:23:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:46Z" name: error-404-isvc-a2dea-predictor.18c007f3c32f20a8 namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: d17e1973-5a87-45b4-9338-9fbad37b340f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:23:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 2ac8595c-2eff-4a4c-a14c-2dd908d68279 kind: Event lastTimestamp: "2026-07-07T14:23:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:46Z" name: error-404-isvc-a2dea-predictor.18c007f3c3ce09de namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: d83c7707-fd4b-4ae1-a14e-55f8aa558f72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:23:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: 2ac8595c-2eff-4a4c-a14c-2dd908d68279 kind: Event lastTimestamp: "2026-07-07T14:24:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:16Z" name: error-404-isvc-a2dea-predictor.18c007f742822afd namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: 37e18e58-2c79-4712-adfe-ae8eb7dadc4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:23:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: 2ac8595c-2eff-4a4c-a14c-2dd908d68279 kind: Event lastTimestamp: "2026-07-07T14:24:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:16Z" name: error-404-isvc-a2dea-predictor.18c007f742ef64af namespace: kserve-ci-e2e-test resourceVersion: "16345" uid: 1ad2eef0-4080-4a22-b46c-73ea8615c237 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a2dea namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 5fd4aba0-ff78-4ae7-9c80-bac551672ecb kind: Event lastTimestamp: "2026-07-07T14:24:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:03Z" name: error-404-isvc-a2dea.18c007f044a8ce37 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 67030953-7068-46f4-816f-f015f9ae6b41 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a2dea namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 5fd4aba0-ff78-4ae7-9c80-bac551672ecb kind: Event lastTimestamp: "2026-07-07T14:24:03Z" message: InferenceService [error-404-isvc-a2dea] is Ready metadata: creationTimestamp: "2026-07-07T14:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:03Z" name: error-404-isvc-a2dea.18c007fedbad3010 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 947e220d-de0d-4758-989e-aca3c271b1fb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:25:31.273965Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ab362-predictor-56447f4fd8-2k4qr to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c0081343d24a15 namespace: kserve-ci-e2e-test resourceVersion: "17311" uid: af4b6bfe-2a1d-45b6-b7d3-3361ece1e070 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17317" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c008135d1ef35c namespace: kserve-ci-e2e-test resourceVersion: "17344" uid: f0c44e3e-6834-4bf2-98e3-8d6d51de20d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c008135f2aa58f namespace: kserve-ci-e2e-test resourceVersion: "17346" uid: 566b9043-80b6-47a0-beaa-2cc707ade9e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c00813620f59e8 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: dbe317cb-9663-42f7-8f70-0f2b666c1788 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c008136271b2cd namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: e327494c-59c3-42e5-87f4-e3eb84047604 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c00813627a7ba2 namespace: kserve-ci-e2e-test resourceVersion: "17349" uid: 4f81b6ce-1d3c-4528-b8db-00eb6b1aa203 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c00813651cea83 namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: 3ef8b953-932e-4c1a-9866-bd6204016722 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c008136577a45c namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: 0d4ac801-201d-4367-b519-7d631bb7cabb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T14:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:34:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:48Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c00813788d0865 namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: c611c07f-01d1-47fe-a5fd-040857d5f70f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:45Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c0089466d11e3b namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 1f3e0573-86e8-4543-a381-43d2c910ce78 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:45Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c0089466d1b4aa namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 86e4a450-7ebd-42ff-9674-8454458f5ece reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: dda11bab-431b-4acd-b3c1-bc0b57459f9a kind: Event lastTimestamp: "2026-07-07T14:34:48Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:34:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:48Z" name: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr.18c00894ecf65fa4 namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: bf0b6000-ddf1-4c25-a42f-7f5ef1f30e46 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ab362-predictor-56447f4fd8 namespace: kserve-ci-e2e-test resourceVersion: "17288" uid: d70150fb-afa7-437c-b66f-e6bfc72584ca kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: 'Created pod: error-404-isvc-ab362-predictor-56447f4fd8-2k4qr' metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor-56447f4fd8.18c0081343276393 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 440355d3-e19b-4f60-b5cd-05012a6928a9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ab362-predictor namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: f0487509-0042-4ebc-84d7-e47b8174b77d kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Scaled up replica set error-404-isvc-ab362-predictor-56447f4fd8 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:31Z" name: error-404-isvc-ab362-predictor.18c00813406366f4 namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: b0d90dc6-9ebc-4c34-8eb3-ecbb7f2c09e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab362-predictor namespace: kserve-ci-e2e-test resourceVersion: "17287" uid: 0f3486ce-00a3-4856-a2b3-3b2dabf2278e kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:31Z" name: error-404-isvc-ab362-predictor.18c00816be827aa0 namespace: kserve-ci-e2e-test resourceVersion: "17920" uid: 91798630-6489-42e5-acf0-d0a37e7112a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab362-predictor namespace: kserve-ci-e2e-test resourceVersion: "17287" uid: 0f3486ce-00a3-4856-a2b3-3b2dabf2278e kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:31Z" name: error-404-isvc-ab362-predictor.18c00816bf2d98f4 namespace: kserve-ci-e2e-test resourceVersion: "17921" uid: 0e188be9-9604-4d29-b869-9661b5fc0cdf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17265" uid: 4ceb3cd6-f21d-45c8-ba0e-f3f24717e7e9 kind: Event lastTimestamp: "2026-07-07T14:26:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:26:18Z" name: error-404-isvc-ab362.18c008134019dd2b namespace: kserve-ci-e2e-test resourceVersion: "17791" uid: 6b5ce283-84e6-4b34-93b9-489f30464187 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17789" uid: 4ceb3cd6-f21d-45c8-ba0e-f3f24717e7e9 kind: Event lastTimestamp: "2026-07-07T14:26:18Z" message: InferenceService [error-404-isvc-ab362] is Ready metadata: creationTimestamp: "2026-07-07T14:26:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:26:18Z" name: error-404-isvc-ab362.18c0081e342509a3 namespace: kserve-ci-e2e-test resourceVersion: "17790" uid: 60db77d5-2571-42a4-8762-dbd0d5ce3e33 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:35:56.123401Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22557" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c008a4bfc00b10 namespace: kserve-ci-e2e-test resourceVersion: "22561" uid: d646b49d-ce87-4944-9b93-fba3fb9f3f8c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22567" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c008a4d96dbfdc namespace: kserve-ci-e2e-test resourceVersion: "22582" uid: fb9555f6-f9e8-4fcd-938f-5f6b134568e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c008a4daf12d9a namespace: kserve-ci-e2e-test resourceVersion: "22584" uid: cbb07c90-555d-40c5-a287-0872c0881207 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c008a4ddd4d3ee namespace: kserve-ci-e2e-test resourceVersion: "22585" uid: 58ccfbf0-946d-4fd7-bcac-c8a3140855e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c008a4de3d6acc namespace: kserve-ci-e2e-test resourceVersion: "22586" uid: 278f6d40-56ca-44a0-b485-b6cca172358a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c008a4de472017 namespace: kserve-ci-e2e-test resourceVersion: "22587" uid: da53fb02-fbbb-4bb9-9aaa-758d4e393f7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c008a4e19d6c6a namespace: kserve-ci-e2e-test resourceVersion: "22589" uid: b9f5c0fb-59a1-4c65-ae97-9aa937c419e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c008a4e206f287 namespace: kserve-ci-e2e-test resourceVersion: "22590" uid: 38375183-3edc-450a-a902-54f42c0bc3e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T14:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:45:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:13Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c008a4f723f2af namespace: kserve-ci-e2e-test resourceVersion: "27184" uid: 290fa389-d8ba-46f1-b03f-27c9c0b35440 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:45:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:11Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c00925f76ecdf7 namespace: kserve-ci-e2e-test resourceVersion: "27129" uid: cd85c5dd-a9bc-4fb2-9734-ffde1cd40739 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:45:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:11Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c00925f76f3038 namespace: kserve-ci-e2e-test resourceVersion: "27130" uid: 2bf208c6-1fc9-44b0-87dd-360c0f93c7a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 480cce6f-bfb8-4409-b5bd-38c19c774e6c kind: Event lastTimestamp: "2026-07-07T14:45:13Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:13Z" name: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7.18c009266b83cfad namespace: kserve-ci-e2e-test resourceVersion: "27183" uid: 09206b9d-8254-4987-9b16-6d375207d6d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f31ce-predictor-85665c44b4 namespace: kserve-ci-e2e-test resourceVersion: "22549" uid: 17d9c126-d224-48a7-846f-21e93175d220 kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: 'Created pod: error-404-isvc-f31ce-predictor-85665c44b4-nq6c7' metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor-85665c44b4.18c008a4bf335f02 namespace: kserve-ci-e2e-test resourceVersion: "22558" uid: 11da2e87-800e-456f-b262-e72f912f8624 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22546" uid: 898616ce-cbb6-40cc-abd7-38f1ea809be4 kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: Scaled up replica set error-404-isvc-f31ce-predictor-85665c44b4 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce-predictor.18c008a4bd78aefb namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: da74936f-6cc7-4636-8253-73291ea64777 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22548" uid: e6e90738-9609-43cd-8f0e-f14a440aa04b kind: Event lastTimestamp: "2026-07-07T14:36:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:11Z" name: error-404-isvc-f31ce-predictor.18c008a83bc8f475 namespace: kserve-ci-e2e-test resourceVersion: "22771" uid: 5f743810-19a0-496a-bf70-c81c4dc83c02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22548" uid: e6e90738-9609-43cd-8f0e-f14a440aa04b kind: Event lastTimestamp: "2026-07-07T14:36:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:11Z" name: error-404-isvc-f31ce-predictor.18c008a83c2be767 namespace: kserve-ci-e2e-test resourceVersion: "22773" uid: 54231ca6-a375-4fda-877c-e50c0da07a6e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22772" uid: e6e90738-9609-43cd-8f0e-f14a440aa04b kind: Event lastTimestamp: "2026-07-07T14:37:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:37:11Z" name: error-404-isvc-f31ce-predictor.18c008abbae373be namespace: kserve-ci-e2e-test resourceVersion: "23280" uid: d0fb8141-b0ba-4a43-8534-56523e2aeaf5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22772" uid: e6e90738-9609-43cd-8f0e-f14a440aa04b kind: Event lastTimestamp: "2026-07-07T14:37:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:37:11Z" name: error-404-isvc-f31ce-predictor.18c008abbb53f1ed namespace: kserve-ci-e2e-test resourceVersion: "23281" uid: 6cc7c49c-2540-4713-a656-ebdf837b4aca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f31ce namespace: kserve-ci-e2e-test resourceVersion: "22531" uid: d664ce77-920e-48da-8685-ebfbf1298971 kind: Event lastTimestamp: "2026-07-07T14:36:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:36:43Z" name: error-404-isvc-f31ce.18c008a4bd74f475 namespace: kserve-ci-e2e-test resourceVersion: "23032" uid: 53d2bb95-61f7-472e-a417-edd5495de6c5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f31ce namespace: kserve-ci-e2e-test resourceVersion: "22531" uid: d664ce77-920e-48da-8685-ebfbf1298971 kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-f31ce": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f31ce": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce.18c008a4bdf7d6d7 namespace: kserve-ci-e2e-test resourceVersion: "22554" uid: d0c774a0-1697-42bf-890d-bc9d7ba80b84 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f31ce namespace: kserve-ci-e2e-test resourceVersion: "22531" uid: d664ce77-920e-48da-8685-ebfbf1298971 kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f31ce": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:56Z" name: error-404-isvc-f31ce.18c008a4bdf81a34 namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: 25496c3b-4a55-45be-9491-b3a8d2290b87 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f31ce namespace: kserve-ci-e2e-test resourceVersion: "23028" uid: d664ce77-920e-48da-8685-ebfbf1298971 kind: Event lastTimestamp: "2026-07-07T14:36:43Z" message: InferenceService [error-404-isvc-f31ce] is Ready metadata: creationTimestamp: "2026-07-07T14:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:36:43Z" name: error-404-isvc-f31ce.18c008afb32c3520 namespace: kserve-ci-e2e-test resourceVersion: "23029" uid: 7bedb010-239a-46fd-b21c-791f8f9a82ee reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:34:45.855918Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fbc26-predictor-78ddb55798-rjlzh to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:34:45Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c00894637b51e2 namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: 41710898-24ae-4b5a-a19e-72cb0e80b058 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fbc26-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c008947218ae43 namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: 16afa478-8f88-495e-acf0-6435cecf6ed3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:34:46Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c00894a0bf9d90 namespace: kserve-ci-e2e-test resourceVersion: "21629" uid: e31f1de7-2f15-4b2e-a744-621d00f0961f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c00894a23ea4cf namespace: kserve-ci-e2e-test resourceVersion: "21631" uid: 43925fab-1c50-4b97-ba7f-1fe2d0dc26c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c00894a52fe371 namespace: kserve-ci-e2e-test resourceVersion: "21632" uid: dc7fc3de-f1d7-4c24-a5fb-bad5a3128fd1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c00894a5b9fc14 namespace: kserve-ci-e2e-test resourceVersion: "21633" uid: 143b8a8c-3583-4f04-b846-325772419248 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c00894a5c4056f namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: 573c870c-a701-4238-9790-0bf6a7fa3dd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:34:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:47Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c00894a879c74b namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: 8fb2ae9b-2c5f-4b86-a184-4c2a8cb5ed60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:34:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:47Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c00894a8e49df9 namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: 86eba9dd-9f5e-472e-9ffc-1505735fb0ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:34:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:35:23Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:23Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c00894d2203bcb namespace: kserve-ci-e2e-test resourceVersion: "22142" uid: 53535832-d928-4fee-9001-f9ef47d79b8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:35:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:55Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c008a4b5dbce48 namespace: kserve-ci-e2e-test resourceVersion: "22500" uid: 2785f5ab-6789-4b3c-ae70-0e5d3575872e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:35:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:55Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c008a4b5dc3af4 namespace: kserve-ci-e2e-test resourceVersion: "22503" uid: 2be177f1-b62e-4899-afbb-77475792ec1d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: e3dba091-791a-4f6a-9d67-881370dd1aa2 kind: Event lastTimestamp: "2026-07-07T14:35:58Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:58Z" name: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh.18c008a55a4fffec namespace: kserve-ci-e2e-test resourceVersion: "22622" uid: 78429af1-4b99-4f40-ba4c-e63c23006b3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fbc26-predictor-78ddb55798 namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: 57f6af83-38f2-4280-8175-1f93fda2fd93 kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: 'Created pod: error-404-isvc-fbc26-predictor-78ddb55798-rjlzh' metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:45Z" name: error-404-isvc-fbc26-predictor-78ddb55798.18c0089462e5330d namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: 71bd85ee-347a-4d0b-80e8-debb448d4733 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: f981842b-9696-41df-9dda-1e99b029d434 kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: Scaled up replica set error-404-isvc-fbc26-predictor-78ddb55798 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:45Z" name: error-404-isvc-fbc26-predictor.18c0089460fb9d09 namespace: kserve-ci-e2e-test resourceVersion: "21570" uid: 3f4fc119-26f8-4e68-b3e2-15ccedb24bc2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:35:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 461c83db-0d9a-4076-98f1-1db50f8a87b7 kind: Event lastTimestamp: "2026-07-07T14:35:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:45Z" name: error-404-isvc-fbc26-predictor.18c00897df8a2622 namespace: kserve-ci-e2e-test resourceVersion: "22359" uid: b66489ee-6efa-43f6-9337-08d1410ab645 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:35:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 461c83db-0d9a-4076-98f1-1db50f8a87b7 kind: Event lastTimestamp: "2026-07-07T14:35:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:45Z" name: error-404-isvc-fbc26-predictor.18c00897dff4bce0 namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: b083ffab-f6ef-48bd-b12c-b4777706a249 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 461c83db-0d9a-4076-98f1-1db50f8a87b7 kind: Event lastTimestamp: "2026-07-07T14:35:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:15Z" name: error-404-isvc-fbc26-predictor.18c0089b5eb12b27 namespace: kserve-ci-e2e-test resourceVersion: "22076" uid: 5505c5c7-b71e-4078-aa89-021ff2a0df8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 461c83db-0d9a-4076-98f1-1db50f8a87b7 kind: Event lastTimestamp: "2026-07-07T14:35:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:15Z" name: error-404-isvc-fbc26-predictor.18c0089b5f2958dc namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: f32d26d0-bd2b-4fd0-9d52-35873ae9aa58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: d8120b91-400f-4351-b429-7370c1d36bab kind: Event lastTimestamp: "2026-07-07T14:35:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:33Z" name: error-404-isvc-fbc26.18c0089460fa2a0a namespace: kserve-ci-e2e-test resourceVersion: "22230" uid: df330482-0059-49ad-b098-e3f8cc88ba53 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: d8120b91-400f-4351-b429-7370c1d36bab kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-fbc26": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fbc26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:34:45Z" name: error-404-isvc-fbc26.18c00894617d9311 namespace: kserve-ci-e2e-test resourceVersion: "21573" uid: d5f604fb-14b9-4b76-b251-a34527901ffe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: d8120b91-400f-4351-b429-7370c1d36bab kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fbc26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:34:45Z" name: error-404-isvc-fbc26.18c00894617dd148 namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: d8af5a47-1120-482f-890d-85b2ab9484f7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "22228" uid: d8120b91-400f-4351-b429-7370c1d36bab kind: Event lastTimestamp: "2026-07-07T14:35:33Z" message: InferenceService [error-404-isvc-fbc26] is Ready metadata: creationTimestamp: "2026-07-07T14:35:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:33Z" name: error-404-isvc-fbc26.18c0089f900f3d63 namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: c81d7076-ab6e-4836-a011-5cb43afb0b01 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:35:06.409653Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fe15b-predictor-957bff694-hw6p5 to ip-10-0-130-145.ec2.internal metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:35:06Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c008992c947a89 namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: dfcba4df-3b2b-4dff-b451-a9ed34973838 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fe15b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:06Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c008993b68a7a6 namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: b75f2748-136f-4752-8c7e-729c4704b33a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21944" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:35:07Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c0089969e85420 namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: d9fcc67f-5c36-4089-ae1b-4d3db2499d93 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c008996b866f58 namespace: kserve-ci-e2e-test resourceVersion: "21980" uid: e9d8ce81-64ae-4c31-b090-918aa1ba90c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c008996eb29350 namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 9c4c5623-d465-45f3-afcc-54a50ebcb398 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c008996f1d30ed namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: 24d0b79f-8586-48ca-8eb1-fcbe2447416b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c008996f25ac22 namespace: kserve-ci-e2e-test resourceVersion: "21985" uid: b6c4a6c8-4981-465b-9ae5-2f23fd03fa51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c008997209632b namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 1ec21d4f-42d7-4cbf-864a-54dacfb3f588 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c0089972791234 namespace: kserve-ci-e2e-test resourceVersion: "21988" uid: 3d44984b-fe87-4138-bafb-a5fe5b725165 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:44:23Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:23Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c00899804cfa03 namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: c5392967-ed5d-4cc3-b06e-ffb2f925ef1e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c0091a64d1649f namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 81cb0fa9-97c4-4d0e-8fa9-d71b2cfce610 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c0091a64d88fb5 namespace: kserve-ci-e2e-test resourceVersion: "26592" uid: 147bf0ed-4593-4aa0-959c-5372090e57dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe15b-predictor-957bff694-hw6p5 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: 824450cf-0ca7-41f8-9099-041a11e42b0a kind: Event lastTimestamp: "2026-07-07T14:44:23Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:44:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:23Z" name: error-404-isvc-fe15b-predictor-957bff694-hw6p5.18c0091af4c0f658 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 009deb39-ab33-4c0c-ac73-921e2cdf02fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-145.ec2.internal source: component: kubelet host: ip-10-0-130-145.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fe15b-predictor-957bff694 namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: d0af80bd-89e9-4cee-b382-22cfb0cf2896 kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: 'Created pod: error-404-isvc-fe15b-predictor-957bff694-hw6p5' metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:06Z" name: error-404-isvc-fe15b-predictor-957bff694.18c008992c1d185a namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 05525d0b-49bf-45da-9b0b-2492e394a6a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: db2079db-e0cb-41d7-9c9c-fb9db7029687 kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: Scaled up replica set error-404-isvc-fe15b-predictor-957bff694 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:06Z" name: error-404-isvc-fe15b-predictor.18c008992a85446d namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: 3dca49aa-9785-44fc-bccd-b9d5d6158bcc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: 6bc23a68-e33f-476f-bfae-c98da1964787 kind: Event lastTimestamp: "2026-07-07T14:35:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:36Z" name: error-404-isvc-fe15b-predictor.18c0089ca8bbf591 namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 53c50d4b-8f6a-4ec5-be6d-c269070cf25f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: 6bc23a68-e33f-476f-bfae-c98da1964787 kind: Event lastTimestamp: "2026-07-07T14:35:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:36Z" name: error-404-isvc-fe15b-predictor.18c0089ca93008ef namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: 5f61c63a-90e1-477b-b65c-2d2dadd4da07 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:35:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22126" uid: 6bc23a68-e33f-476f-bfae-c98da1964787 kind: Event lastTimestamp: "2026-07-07T14:36:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:21Z" name: error-404-isvc-fe15b-predictor.18c008a3a68ba5f6 namespace: kserve-ci-e2e-test resourceVersion: "22855" uid: 4b67575f-4028-427d-9207-1e93923e95c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:35:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22126" uid: 6bc23a68-e33f-476f-bfae-c98da1964787 kind: Event lastTimestamp: "2026-07-07T14:36:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:21Z" name: error-404-isvc-fe15b-predictor.18c008a3a6f9d2bd namespace: kserve-ci-e2e-test resourceVersion: "22856" uid: f6d01ed4-7781-4920-92dd-b958db0d2dec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe15b namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: ac04a8bc-842b-48e6-87ff-bc20056b25ec kind: Event lastTimestamp: "2026-07-07T14:35:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:53Z" name: error-404-isvc-fe15b.18c008992a4c30ac namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: b92eda22-bdbb-46a5-b62b-d7f18afbefbe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe15b namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: ac04a8bc-842b-48e6-87ff-bc20056b25ec kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-fe15b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fe15b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:06Z" name: error-404-isvc-fe15b.18c008992abb191e namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 138d3fea-6d56-484a-be36-0cfe364e36c3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe15b namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: ac04a8bc-842b-48e6-87ff-bc20056b25ec kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fe15b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:06Z" name: error-404-isvc-fe15b.18c008992abb52e1 namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: 9f4b839d-b39b-489f-88ed-9b2268ea3a7f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe15b namespace: kserve-ci-e2e-test resourceVersion: "22441" uid: ac04a8bc-842b-48e6-87ff-bc20056b25ec kind: Event lastTimestamp: "2026-07-07T14:35:53Z" message: InferenceService [error-404-isvc-fe15b] is Ready metadata: creationTimestamp: "2026-07-07T14:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:53Z" name: error-404-isvc-fe15b.18c008a43bad1823 namespace: kserve-ci-e2e-test resourceVersion: "22442" uid: c257c91d-d89d-4be9-8920-7c8e3456f99c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:23:00.913288Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-875bcd459-2cg9t to ip-10-0-130-152.ec2.internal metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:23:00Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f041a08166 namespace: kserve-ci-e2e-test resourceVersion: "15525" uid: 9ed59447-110e-438d-b1c1-1cfde0721005 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f052719a2e namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 728ccdbf-918d-4d32-8326-caf102d7e0f4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f0805b8911 namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: ec367006-5dee-4d78-8212-7233bd31d4ff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f0821f892d namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: ccecc42b-8c2f-4e0f-b29d-8520fc842afe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:02Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f085a4101d namespace: kserve-ci-e2e-test resourceVersion: "15655" uid: 27af36c2-a728-493d-8265-368dfafa1827 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:02Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f086253b74 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: f3f43975-2d71-4145-b7ce-ca48381816e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:06Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-07T14:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:06Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f1811f553c namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: 690baa3f-92e4-401c-806a-776141855526 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:33Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 27.593s (27.593s including waiting). Image size: 1555845045 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:33Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f7edce612d namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 6f66febb-c725-40f7-bc5d-7119a64055a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:23:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:33Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f7f3061fcb namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: 26618592-d075-49b9-b811-c409980c348c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:23:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:33Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f7f43b25d9 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: ca960d42-9a5b-4275-87be-e65d8997f07e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:33Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T14:23:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:33Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f7f44ae985 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 39783285-4559-487f-8af3-1e13d57fb804 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.217s (3.217s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:37Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f8b4174ee0 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: f1879d01-d052-4cec-8459-ff8690475e9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:37Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f8b9dad2cb namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: f2194f05-83c8-4d16-8cf2-bf55f04a110a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:23:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:37Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f8ba57433d namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 1645c81d-4a5e-4b23-859e-6292b9103f4a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:24:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:50Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c007f8c6263fb9 namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: a22e2e58-dc76-4477-9bae-f32b9853ff29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c0081339cdf600 namespace: kserve-ci-e2e-test resourceVersion: "17259" uid: 49593c37-bd0d-4b90-8f70-eaf6b66aec48 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c0081339ce5dcc namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: eff2d3e1-ca3a-43c0-a163-f3064f45cb08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: e22a677a-65e5-4ab3-8647-6c0bdf612c1a kind: Event lastTimestamp: "2026-07-07T14:25:33Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:33Z" name: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t.18c00813c88aedce namespace: kserve-ci-e2e-test resourceVersion: "17381" uid: ff3b7e52-c5a3-4afd-81c0-a8fb3fa39dcb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-875bcd459 namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 8baf0121-9fde-4d44-89d5-01600ca3e454 kind: Event lastTimestamp: "2026-07-07T14:23:00Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-875bcd459-2cg9t' metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:00Z" name: isvc-sklearn-graph-1-predictor-875bcd459.18c007f041205484 namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: b3e23261-6f48-4f11-a69e-dac9b1894a4c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: f843bab8-9402-4f44-873b-574f77a8ed5d kind: Event lastTimestamp: "2026-07-07T14:23:00Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-875bcd459 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:00Z" name: isvc-sklearn-graph-1-predictor.18c007f03e1a302a namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: a433908c-aa18-4c40-a31d-7c2f6f422148 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:23:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: 93e14a0a-6bef-4594-a073-d8e12ff98ac7 kind: Event lastTimestamp: "2026-07-07T14:23:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:45Z" name: isvc-sklearn-graph-1-predictor.18c007f3bcdca2d8 namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: a79de1c9-eb80-433b-874a-3cab134096ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:23:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: 93e14a0a-6bef-4594-a073-d8e12ff98ac7 kind: Event lastTimestamp: "2026-07-07T14:23:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:45Z" name: isvc-sklearn-graph-1-predictor.18c007f3bd66a8a6 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 29bc77fa-cd5d-4b9c-b926-5632b442b010 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:24:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 93e14a0a-6bef-4594-a073-d8e12ff98ac7 kind: Event lastTimestamp: "2026-07-07T14:25:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:15Z" name: isvc-sklearn-graph-1-predictor.18c007fe39cebd2b namespace: kserve-ci-e2e-test resourceVersion: "17029" uid: d6859266-e7bb-441f-9341-bba14b2b348d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:24:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 93e14a0a-6bef-4594-a073-d8e12ff98ac7 kind: Event lastTimestamp: "2026-07-07T14:25:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:16Z" name: isvc-sklearn-graph-1-predictor.18c007fe3a4e603c namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: b04ed8ec-e4d5-4de9-93fc-8b5a9c8abae9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: 8f848ffb-7ea0-4de5-8911-e12f4a7e6bff kind: Event lastTimestamp: "2026-07-07T14:25:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:00Z" name: isvc-sklearn-graph-1.18c007f03e1d2d90 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: f3b446a5-b7b8-4166-b39f-a8a03b299cbf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 8f848ffb-7ea0-4de5-8911-e12f4a7e6bff kind: Event lastTimestamp: "2026-07-07T14:25:00Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-07T14:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:00Z" name: isvc-sklearn-graph-1.18c0080c07873351 namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: b227acc4-8cab-47ca-959a-8a9727ce8e58 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:23:01.304327Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 to ip-10-0-130-152.ec2.internal metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f058ef493a namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: eca33d9f-2fe2-4fdc-a019-b16fe27f57f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f073160e6f namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 35d68ccc-b709-4630-8a43-1c3d5231a433 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f074f45a3f namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 209b0ca7-e7a2-48da-9df8-f4da5d7976a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f07803e8aa namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: 72808bc3-e4de-42c1-93b0-f8d04acb45da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f0789ba280 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 040260e1-51e6-4009-88c8-4e28e279df3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:06Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-07T14:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:06Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f180eb9445 namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: 5028e2c8-6016-45ca-a0e1-3ead440da64e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:34Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 27.97s (27.97s including waiting). Image size: 1555845045 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:34Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f8041f9139 namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 52037cdb-b0bc-47ac-b71d-a5c4f286d6a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:34Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f80a6871eb namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: e034e1b4-32fb-4c5b-9c7b-0879bd2cd3d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:34Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f80b8aa460 namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 4815c8b9-0988-4ab8-b420-d5bc55a6bc39 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T14:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:34Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f80b993c9b namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 79752a62-db6d-4b64-b110-876a16772772 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.837s (2.837s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:37Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f8b4b61d7b namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: ad6e9bf9-615c-464b-bcb4-d200a8f7b8f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:37Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f8b9dc3848 namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: ea48d6d1-da49-4d8d-994a-3a6d899a0200 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:23:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:37Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f8ba4af18f namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: cb6d1954-e52f-49a3-a996-166ea9eb71ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:23:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:25:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:23:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:34Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c007f9020ac19f namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: d9d0fd51-a66b-4aac-b477-cc4c3371c044 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c008134d4a5de5 namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: e54e9afa-130b-4ba4-a548-2428c81868ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c008134d4aeb0f namespace: kserve-ci-e2e-test resourceVersion: "17339" uid: 66936ce4-dcb7-4f62-9e83-3799811dcf6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 4d1eb367-5c69-4cab-8034-50ac36dc77ee kind: Event lastTimestamp: "2026-07-07T14:25:34Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:25:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:34Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7.18c00814046c1b76 namespace: kserve-ci-e2e-test resourceVersion: "17391" uid: fa5c44d7-0d87-4c74-9582-33cda830a397 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6c8478f9b5 namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 73eaa1f6-c3a7-4927-a5de-856d3da02653 kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6c8478f9b5-rffx7' metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5.18c007f0586d84f3 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: c7ccf49b-6f0d-4737-b98b-b38c5bf8f7df reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 69fb6f2d-8b45-41c1-ad09-7c906d3b99b2 kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6c8478f9b5 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:01Z" name: isvc-sklearn-graph-2-predictor.18c007f053b95337 namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: 629da75e-97e8-4939-8219-685377537c2d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 1065bfb2-09a0-4de6-b71d-e21e139cdcfc kind: Event lastTimestamp: "2026-07-07T14:24:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:16Z" name: isvc-sklearn-graph-2-predictor.18c007f3d221315a namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: e8b3281b-44ba-4a95-a032-6c1ea388bb4d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 1065bfb2-09a0-4de6-b71d-e21e139cdcfc kind: Event lastTimestamp: "2026-07-07T14:24:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:16Z" name: isvc-sklearn-graph-2-predictor.18c007f3d289716a namespace: kserve-ci-e2e-test resourceVersion: "16352" uid: 7a828eeb-4508-4108-8c17-af0f396ef90c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 1065bfb2-09a0-4de6-b71d-e21e139cdcfc kind: Event lastTimestamp: "2026-07-07T14:25:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:16Z" name: isvc-sklearn-graph-2-predictor.18c008054c430ca2 namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: 559af3c9-d49f-4710-82b9-be2c3bf62223 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 1065bfb2-09a0-4de6-b71d-e21e139cdcfc kind: Event lastTimestamp: "2026-07-07T14:25:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:16Z" name: isvc-sklearn-graph-2-predictor.18c008054cc28bd6 namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: c16a634f-49d2-4447-80be-99faf34fa224 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: 9c4a7dc2-2a0b-45bf-b264-764de81abbc5 kind: Event lastTimestamp: "2026-07-07T14:24:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:54Z" name: isvc-sklearn-graph-2.18c007f053c37274 namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: 951fedc7-5cad-4755-a594-b23337641d11 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 9c4a7dc2-2a0b-45bf-b264-764de81abbc5 kind: Event lastTimestamp: "2026-07-07T14:24:54Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-07T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:54Z" name: isvc-sklearn-graph-2.18c0080abcf8ac48 namespace: kserve-ci-e2e-test resourceVersion: "16860" uid: a2eab353-5f95-46a7-8ba4-f5c880276a56 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:23:01.146732Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-d2s8g to ip-10-0-130-152.ec2.internal metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:23:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f04f8a807b namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 15ede9ff-f0df-4f64-b9df-9aada487fc05 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:23:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f069774cc9 namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c2a76bd2-f85e-4dcf-a5b8-9dac11a1bebd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f06b2a4cf9 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: daccd11d-fdc2-4fb0-a725-27a04878cdb2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f06e3241f0 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: aaa3cd26-af2e-4402-b78a-fb02ed8a650b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f06ed189da namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: a794df87-edf1-4e35-8a2b-9bfcb9f2ac25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:06Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-07T14:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f18109bee7 namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: 881fb560-2cab-45c5-8216-f340cfdc5a5c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:34Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 28.373s (28.373s including waiting). Image size: 1305547511 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f81c40e313 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: cf66aa9d-9cfb-45ce-8a5c-0f9cd365f2b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f821e3192a namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 1999c680-4aad-4662-b9eb-282e076d0dc5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f82370c0af namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 7a88e68e-7d6d-4b5a-8a45-ed7d66b9bf60 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T14:23:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f823818871 namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 85dc6698-205a-4db6-a796-4cea18874f98 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.427s (2.427s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f8b4327f8a namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 88441957-0a1d-49df-a483-4804ec48ede3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f8b9dc9a63 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 706b6714-4455-4975-bcde-570738c370cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:23:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:23:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f8ba57f800 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: cea971b5-8519-4e2e-b3aa-3c7cb87e9447 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T14:23:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:25:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:23:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c007f9020af89b namespace: kserve-ci-e2e-test resourceVersion: "17392" uid: e8f6c8e0-33e1-4882-8e99-0423b97f5f12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c0081341dbbbdd namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: 8010b277-5dfd-4e7f-b022-5fcfa64f1ede reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c0081341dccf3d namespace: kserve-ci-e2e-test resourceVersion: "17302" uid: c0774cc5-27d5-43d5-b5e3-82d437bfedde reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 743150c8-c506-4882-a72e-4a222b225ddf kind: Event lastTimestamp: "2026-07-07T14:25:34Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:25:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-d2s8g.18c00814046b08d0 namespace: kserve-ci-e2e-test resourceVersion: "17390" uid: 107f26d7-61ed-45f9-9e9b-a143d65841f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-152.ec2.internal source: component: kubelet host: ip-10-0-130-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 3715b1eb-740e-4680-a6cf-11be045739ca kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-d2s8g' metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:01Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c007f04f19df1e namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 11d1d099-abc5-4076-af7f-5dbf500789de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: b47130e6-1120-4de5-ba56-6bac846ee9f1 kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:01Z" name: isvc-xgboost-graph-predictor.18c007f04b814928 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 49180e0e-838c-4e16-8b1e-64ae800cd731 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: 965376a3-5f69-473e-8fa2-b288bfd718b0 kind: Event lastTimestamp: "2026-07-07T14:24:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:01Z" name: isvc-xgboost-graph-predictor.18c007f3ca23d797 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 535d47d9-d42b-4945-a9ef-9784c5997472 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: 965376a3-5f69-473e-8fa2-b288bfd718b0 kind: Event lastTimestamp: "2026-07-07T14:24:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:01Z" name: isvc-xgboost-graph-predictor.18c007f3ca95aca6 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 76aa6584-4594-4a4e-bbe5-67e23c90414b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:24:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 965376a3-5f69-473e-8fa2-b288bfd718b0 kind: Event lastTimestamp: "2026-07-07T14:25:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:16Z" name: isvc-xgboost-graph-predictor.18c00801c63d21e7 namespace: kserve-ci-e2e-test resourceVersion: "17037" uid: ac61a909-1925-441b-aded-8902336f9a9a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:24:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 965376a3-5f69-473e-8fa2-b288bfd718b0 kind: Event lastTimestamp: "2026-07-07T14:25:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:16Z" name: isvc-xgboost-graph-predictor.18c00801c6a62536 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: dc557bdf-da68-4aff-9c17-d8a3220c1d6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: 11b8ffa4-0d54-4ae7-bce6-53066b70e9fb kind: Event lastTimestamp: "2026-07-07T14:24:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:44Z" name: isvc-xgboost-graph.18c007f04b6bc492 namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 35849a73-3087-4a7b-8dce-cbe0b21c3ea3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16759" uid: 11b8ffa4-0d54-4ae7-bce6-53066b70e9fb kind: Event lastTimestamp: "2026-07-07T14:24:44Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-07T14:24:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:44Z" name: isvc-xgboost-graph.18c0080867b29d79 namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: f9be8e75-6963-42f1-856e-f18a0fcb1fc2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:25:20.727687Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-fcb978798-2vdlf namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 7fd44486-5832-447e-a7da-ed2a1be0f4bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-fcb978798-2vdlf to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:25:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:25:20Z" name: model-chainer-fcb978798-2vdlf.18c00810cf36d434 namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: 343279cc-7c6a-430a-9539-112fbb6b57ac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-fcb978798-2vdlf namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: 7fd44486-5832-447e-a7da-ed2a1be0f4bb kind: Event lastTimestamp: "2026-07-07T14:25:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:20Z" name: model-chainer-fcb978798-2vdlf.18c00810d7b7c454 namespace: kserve-ci-e2e-test resourceVersion: "17114" uid: 729dcd45-a505-4364-967f-38f7e8cae5c2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:21Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-fcb978798-2vdlf namespace: kserve-ci-e2e-test resourceVersion: "17107" uid: 7fd44486-5832-447e-a7da-ed2a1be0f4bb kind: Event lastTimestamp: "2026-07-07T14:25:21Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:25:21Z" name: model-chainer-fcb978798-2vdlf.18c008110c9c4a8a namespace: kserve-ci-e2e-test resourceVersion: "17119" uid: 27474476-204a-45c0-8373-22db1d97f74b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fcb978798-2vdlf namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: 7fd44486-5832-447e-a7da-ed2a1be0f4bb kind: Event lastTimestamp: "2026-07-07T14:25:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:21Z" name: model-chainer-fcb978798-2vdlf.18c008110f70c2ad namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: e82af3cb-d881-4ada-95c5-32a08c3e7614 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fcb978798-2vdlf namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: 7fd44486-5832-447e-a7da-ed2a1be0f4bb kind: Event lastTimestamp: "2026-07-07T14:25:21Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-07T14:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:21Z" name: model-chainer-fcb978798-2vdlf.18c008111276b609 namespace: kserve-ci-e2e-test resourceVersion: "17122" uid: ab215cae-2c81-4825-b4a7-c97a5f18c0a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fcb978798-2vdlf namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: 7fd44486-5832-447e-a7da-ed2a1be0f4bb kind: Event lastTimestamp: "2026-07-07T14:25:21Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-07T14:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:21Z" name: model-chainer-fcb978798-2vdlf.18c0081113097856 namespace: kserve-ci-e2e-test resourceVersion: "17123" uid: 374de837-7b5a-4778-aa84-63fc76945505 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fcb978798-2vdlf namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: 7fd44486-5832-447e-a7da-ed2a1be0f4bb kind: Event lastTimestamp: "2026-07-07T14:25:30Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-07T14:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:30Z" name: model-chainer-fcb978798-2vdlf.18c0081327cff966 namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: dc9592b9-a2a3-43d0-b66c-c945d7505977 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fcb978798-2vdlf namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: 7fd44486-5832-447e-a7da-ed2a1be0f4bb kind: Event lastTimestamp: "2026-07-07T14:25:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:57Z" name: model-chainer-fcb978798-2vdlf.18c00813a372e60f namespace: kserve-ci-e2e-test resourceVersion: "17623" uid: 1f5e0e0c-c9a5-44c3-846d-088e9f5ba902 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-fcb978798 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: c14fa3f2-c46b-4dda-ace9-2620be936251 kind: Event lastTimestamp: "2026-07-07T14:25:20Z" message: 'Created pod: model-chainer-fcb978798-2vdlf' metadata: creationTimestamp: "2026-07-07T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:20Z" name: model-chainer-fcb978798.18c00810ce625869 namespace: kserve-ci-e2e-test resourceVersion: "17100" uid: bbc6718a-0435-4b57-a423-fdefba2d3b62 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17092" uid: 2d3fac9f-4e5a-49fb-afe6-150f4912430d kind: Event lastTimestamp: "2026-07-07T14:25:20Z" message: Scaled up replica set model-chainer-fcb978798 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:20Z" name: model-chainer.18c00810cd08bcae namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: c9df5f70-1922-484c-ae8f-0f0b7c98e9ce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 9084f4b0-81dc-4d1b-b5f3-dae81ea18e2d kind: Event lastTimestamp: "2026-07-07T14:25:20Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-07T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:20Z" name: model-chainer.18c00810d42e69da namespace: kserve-ci-e2e-test resourceVersion: "17112" uid: 41d94aac-b121-4c7d-8cdb-1f80e5ac931d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:26:31.039267Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ab362-6f9b459797-kj6g5 namespace: kserve-ci-e2e-test resourceVersion: "17900" uid: 5bad35b6-a0bf-4113-bacd-c735a955dd79 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-ab362-6f9b459797-kj6g5 to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:26:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:26:31Z" name: sequence-graph-ab362-6f9b459797-kj6g5.18c008212e1c7ebf namespace: kserve-ci-e2e-test resourceVersion: "17906" uid: 778dd0cf-67c1-4d16-926c-6109f33a67b2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:31Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ab362-6f9b459797-kj6g5 namespace: kserve-ci-e2e-test resourceVersion: "17909" uid: 5bad35b6-a0bf-4113-bacd-c735a955dd79 kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:26:31Z" name: sequence-graph-ab362-6f9b459797-kj6g5.18c0082147f48e1f namespace: kserve-ci-e2e-test resourceVersion: "17922" uid: d2ca70ea-cb9f-494f-bec5-2780531c22a8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ab362} kind: Pod name: sequence-graph-ab362-6f9b459797-kj6g5 namespace: kserve-ci-e2e-test resourceVersion: "17902" uid: 5bad35b6-a0bf-4113-bacd-c735a955dd79 kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:31Z" name: sequence-graph-ab362-6f9b459797-kj6g5.18c00821496d2c44 namespace: kserve-ci-e2e-test resourceVersion: "17924" uid: 1fe73fb1-ed6a-4fb8-98fe-6970fdab053a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ab362} kind: Pod name: sequence-graph-ab362-6f9b459797-kj6g5 namespace: kserve-ci-e2e-test resourceVersion: "17902" uid: 5bad35b6-a0bf-4113-bacd-c735a955dd79 kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: 'Created container: sequence-graph-ab362' metadata: creationTimestamp: "2026-07-07T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:31Z" name: sequence-graph-ab362-6f9b459797-kj6g5.18c008214c360bc0 namespace: kserve-ci-e2e-test resourceVersion: "17925" uid: 60d85a6a-e9bb-4c08-8c9c-a83ca1e21fbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ab362} kind: Pod name: sequence-graph-ab362-6f9b459797-kj6g5 namespace: kserve-ci-e2e-test resourceVersion: "17902" uid: 5bad35b6-a0bf-4113-bacd-c735a955dd79 kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: Started container sequence-graph-ab362 metadata: creationTimestamp: "2026-07-07T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:31Z" name: sequence-graph-ab362-6f9b459797-kj6g5.18c008214cae8888 namespace: kserve-ci-e2e-test resourceVersion: "17926" uid: 7aea7783-a2ac-4e40-bcfb-08a9345c0ea1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ab362} kind: Pod name: sequence-graph-ab362-6f9b459797-kj6g5 namespace: kserve-ci-e2e-test resourceVersion: "17902" uid: 5bad35b6-a0bf-4113-bacd-c735a955dd79 kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Stopping container sequence-graph-ab362 metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: sequence-graph-ab362-6f9b459797-kj6g5.18c008946e02268e namespace: kserve-ci-e2e-test resourceVersion: "21607" uid: a826ca46-8b26-48a9-916f-6a4ad2d74add reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:34:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ab362} kind: Pod name: sequence-graph-ab362-6f9b459797-kj6g5 namespace: kserve-ci-e2e-test resourceVersion: "17902" uid: 5bad35b6-a0bf-4113-bacd-c735a955dd79 kind: Event lastTimestamp: "2026-07-07T14:35:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:34:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:13Z" name: sequence-graph-ab362-6f9b459797-kj6g5.18c00894ebe0a567 namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 15c9a5f7-96f8-470a-83aa-a8e2e3523a1a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-ab362-6f9b459797 namespace: kserve-ci-e2e-test resourceVersion: "17897" uid: b3c28eab-93f1-412f-bfa7-a543bb170b26 kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: 'Created pod: sequence-graph-ab362-6f9b459797-kj6g5' metadata: creationTimestamp: "2026-07-07T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:31Z" name: sequence-graph-ab362-6f9b459797.18c008212d62c5cd namespace: kserve-ci-e2e-test resourceVersion: "17901" uid: 241cc6e2-1fc5-41b2-aad8-44548c2b7a89 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17895" uid: 9e3ab300-c448-43cd-8bc5-e5ac3af3299b kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: Scaled up replica set sequence-graph-ab362-6f9b459797 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:31Z" name: sequence-graph-ab362.18c008212c9248b1 namespace: kserve-ci-e2e-test resourceVersion: "17898" uid: d0a4bc43-3de4-47c4-adf8-56bb18811fcc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17911" uid: f54c2c1e-09da-4d94-8619-4e5f27f2787f kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: InferenceGraph [sequence-graph-ab362] is Ready metadata: creationTimestamp: "2026-07-07T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:26:31Z" name: sequence-graph-ab362.18c008213227a3c5 namespace: kserve-ci-e2e-test resourceVersion: "17913" uid: b352e761-15cd-435b-94b1-a8a46a8023f3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:26:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17896" uid: f94bec78-fa03-4fda-8217-7d058f3d8f91 kind: Event lastTimestamp: "2026-07-07T14:27:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:27:01Z" name: sequence-graph-ab362.18c00824aaff5299 namespace: kserve-ci-e2e-test resourceVersion: "18144" uid: 24fe70a7-27c4-41fc-801e-842c6d98d505 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:26:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17896" uid: f94bec78-fa03-4fda-8217-7d058f3d8f91 kind: Event lastTimestamp: "2026-07-07T14:27:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:27:01Z" name: sequence-graph-ab362.18c00824ab6b49db namespace: kserve-ci-e2e-test resourceVersion: "18146" uid: 9cab622f-e360-4879-a48d-881545b1dade reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:36:56.118225Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f31ce-9c588fcbb-qjxf8 namespace: kserve-ci-e2e-test resourceVersion: "23142" uid: 4b2a024a-5068-40c0-8124-bc69b169e130 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f31ce-9c588fcbb-qjxf8 to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:36:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:36:56Z" name: sequence-graph-f31ce-9c588fcbb-qjxf8.18c008b2b7b874a3 namespace: kserve-ci-e2e-test resourceVersion: "23147" uid: 303ae90f-72f8-40fc-a0a1-66681ede703a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f31ce-9c588fcbb-qjxf8 namespace: kserve-ci-e2e-test resourceVersion: "23145" uid: 4b2a024a-5068-40c0-8124-bc69b169e130 kind: Event lastTimestamp: "2026-07-07T14:36:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-f31ce-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:56Z" name: sequence-graph-f31ce-9c588fcbb-qjxf8.18c008b2c2c3d721 namespace: kserve-ci-e2e-test resourceVersion: "23159" uid: d2a2a68b-7e7b-4445-bcdd-5ec1393a42b6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f31ce-9c588fcbb-qjxf8 namespace: kserve-ci-e2e-test resourceVersion: "23153" uid: 4b2a024a-5068-40c0-8124-bc69b169e130 kind: Event lastTimestamp: "2026-07-07T14:36:57Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:36:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:36:57Z" name: sequence-graph-f31ce-9c588fcbb-qjxf8.18c008b2f5472e66 namespace: kserve-ci-e2e-test resourceVersion: "23168" uid: 889f1450-c6c5-4c01-a7e6-0bb9b77022da reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f31ce} kind: Pod name: sequence-graph-f31ce-9c588fcbb-qjxf8 namespace: kserve-ci-e2e-test resourceVersion: "23145" uid: 4b2a024a-5068-40c0-8124-bc69b169e130 kind: Event lastTimestamp: "2026-07-07T14:36:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:36:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:57Z" name: sequence-graph-f31ce-9c588fcbb-qjxf8.18c008b2f7341a9e namespace: kserve-ci-e2e-test resourceVersion: "23171" uid: 750c25bb-3b31-4a35-bc05-ca69da220ef7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f31ce} kind: Pod name: sequence-graph-f31ce-9c588fcbb-qjxf8 namespace: kserve-ci-e2e-test resourceVersion: "23145" uid: 4b2a024a-5068-40c0-8124-bc69b169e130 kind: Event lastTimestamp: "2026-07-07T14:36:57Z" message: 'Created container: sequence-graph-f31ce' metadata: creationTimestamp: "2026-07-07T14:36:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:57Z" name: sequence-graph-f31ce-9c588fcbb-qjxf8.18c008b2fa5ae0bb namespace: kserve-ci-e2e-test resourceVersion: "23174" uid: da275b8f-768a-4fe0-af49-fd153bffcb00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f31ce} kind: Pod name: sequence-graph-f31ce-9c588fcbb-qjxf8 namespace: kserve-ci-e2e-test resourceVersion: "23145" uid: 4b2a024a-5068-40c0-8124-bc69b169e130 kind: Event lastTimestamp: "2026-07-07T14:36:57Z" message: Started container sequence-graph-f31ce metadata: creationTimestamp: "2026-07-07T14:36:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:57Z" name: sequence-graph-f31ce-9c588fcbb-qjxf8.18c008b2fad49551 namespace: kserve-ci-e2e-test resourceVersion: "23175" uid: 2f1a1551-0c6c-4c25-8692-2ea1bfbf6e53 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f31ce} kind: Pod name: sequence-graph-f31ce-9c588fcbb-qjxf8 namespace: kserve-ci-e2e-test resourceVersion: "23145" uid: 4b2a024a-5068-40c0-8124-bc69b169e130 kind: Event lastTimestamp: "2026-07-07T14:45:10Z" message: Stopping container sequence-graph-f31ce metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:10Z" name: sequence-graph-f31ce-9c588fcbb-qjxf8.18c00925e77223c0 namespace: kserve-ci-e2e-test resourceVersion: "27040" uid: 43475097-841f-4501-9051-1f653f6da775 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f31ce} kind: Pod name: sequence-graph-f31ce-9c588fcbb-qjxf8 namespace: kserve-ci-e2e-test resourceVersion: "23145" uid: 4b2a024a-5068-40c0-8124-bc69b169e130 kind: Event lastTimestamp: "2026-07-07T14:45:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:38Z" name: sequence-graph-f31ce-9c588fcbb-qjxf8.18c009268ae8d66a namespace: kserve-ci-e2e-test resourceVersion: "27584" uid: fbedafdd-55e3-48b1-84d4-a9c24c6c7740 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f31ce-9c588fcbb namespace: kserve-ci-e2e-test resourceVersion: "23139" uid: 1569940d-86dc-4046-b0e4-144485dc4fdd kind: Event lastTimestamp: "2026-07-07T14:36:56Z" message: 'Created pod: sequence-graph-f31ce-9c588fcbb-qjxf8' metadata: creationTimestamp: "2026-07-07T14:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:56Z" name: sequence-graph-f31ce-9c588fcbb.18c008b2b7441bdb namespace: kserve-ci-e2e-test resourceVersion: "23144" uid: 0071e1fc-1bd1-43e1-8d7e-4602b1c925d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f31ce namespace: kserve-ci-e2e-test resourceVersion: "23137" uid: 270ae38f-b3b8-4900-8fda-8e521591ea3a kind: Event lastTimestamp: "2026-07-07T14:36:56Z" message: Scaled up replica set sequence-graph-f31ce-9c588fcbb from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:56Z" name: sequence-graph-f31ce.18c008b2b61e53df namespace: kserve-ci-e2e-test resourceVersion: "23141" uid: 87b69067-b0c8-4ec5-b152-b63276b3ae47 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f31ce namespace: kserve-ci-e2e-test resourceVersion: "23155" uid: 1506723e-5e48-4de5-978a-ea7fbdf1adc2 kind: Event lastTimestamp: "2026-07-07T14:36:56Z" message: InferenceGraph [sequence-graph-f31ce] is Ready metadata: creationTimestamp: "2026-07-07T14:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:36:56Z" name: sequence-graph-f31ce.18c008b2bc9a4249 namespace: kserve-ci-e2e-test resourceVersion: "23157" uid: 3a436622-e23e-4d5e-b62a-e7e8a3917ff0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:37:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f31ce namespace: kserve-ci-e2e-test resourceVersion: "23138" uid: 025e8375-bce8-4ab8-8d26-94b3015bcb42 kind: Event lastTimestamp: "2026-07-07T14:37:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:37:11Z" name: sequence-graph-f31ce.18c008b634f128e0 namespace: kserve-ci-e2e-test resourceVersion: "23278" uid: 652c68fa-7f0a-414b-a374-c5536dbd0002 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:37:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f31ce namespace: kserve-ci-e2e-test resourceVersion: "23138" uid: 025e8375-bce8-4ab8-8d26-94b3015bcb42 kind: Event lastTimestamp: "2026-07-07T14:37:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:37:11Z" name: sequence-graph-f31ce.18c008b635746b1a namespace: kserve-ci-e2e-test resourceVersion: "23279" uid: 9ebf39cf-3e39-4f22-87e8-70ff0f4293d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:37:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f31ce namespace: kserve-ci-e2e-test resourceVersion: "23277" uid: 025e8375-bce8-4ab8-8d26-94b3015bcb42 kind: Event lastTimestamp: "2026-07-07T14:37:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:37:26Z" name: sequence-graph-f31ce.18c008b9b42a2e17 namespace: kserve-ci-e2e-test resourceVersion: "23390" uid: fb6971d4-4c2b-4ed5-ba80-033ada01978c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:37:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f31ce namespace: kserve-ci-e2e-test resourceVersion: "23277" uid: 025e8375-bce8-4ab8-8d26-94b3015bcb42 kind: Event lastTimestamp: "2026-07-07T14:37:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:37:26Z" name: sequence-graph-f31ce.18c008b9b47a4e5b namespace: kserve-ci-e2e-test resourceVersion: "23393" uid: 7143288c-ba87-4f61-809f-010e5473c584 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:35:45.782262Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fbc26-59cf79489b-jkc4g namespace: kserve-ci-e2e-test resourceVersion: "22342" uid: 441214a3-86a7-4c2c-b1d6-6aef7b931403 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-fbc26-59cf79489b-jkc4g to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:35:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:35:45Z" name: sequence-graph-fbc26-59cf79489b-jkc4g.18c008a2575ec866 namespace: kserve-ci-e2e-test resourceVersion: "22348" uid: 8ad62509-ed9c-422a-8621-39d160a84b75 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fbc26-59cf79489b-jkc4g namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 441214a3-86a7-4c2c-b1d6-6aef7b931403 kind: Event lastTimestamp: "2026-07-07T14:35:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-fbc26-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:45Z" name: sequence-graph-fbc26-59cf79489b-jkc4g.18c008a260452945 namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 05985493-a023-4328-9b51-3e15f86d4049 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fbc26-59cf79489b-jkc4g namespace: kserve-ci-e2e-test resourceVersion: "22352" uid: 441214a3-86a7-4c2c-b1d6-6aef7b931403 kind: Event lastTimestamp: "2026-07-07T14:35:46Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:35:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:35:46Z" name: sequence-graph-fbc26-59cf79489b-jkc4g.18c008a294ac82f8 namespace: kserve-ci-e2e-test resourceVersion: "22372" uid: 118df7b3-5c18-4539-8080-56b63260f74d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbc26} kind: Pod name: sequence-graph-fbc26-59cf79489b-jkc4g namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 441214a3-86a7-4c2c-b1d6-6aef7b931403 kind: Event lastTimestamp: "2026-07-07T14:35:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:46Z" name: sequence-graph-fbc26-59cf79489b-jkc4g.18c008a296008b75 namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: b242f776-ae82-4822-93e8-2455b2810954 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbc26} kind: Pod name: sequence-graph-fbc26-59cf79489b-jkc4g namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 441214a3-86a7-4c2c-b1d6-6aef7b931403 kind: Event lastTimestamp: "2026-07-07T14:35:46Z" message: 'Created container: sequence-graph-fbc26' metadata: creationTimestamp: "2026-07-07T14:35:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:46Z" name: sequence-graph-fbc26-59cf79489b-jkc4g.18c008a299f323aa namespace: kserve-ci-e2e-test resourceVersion: "22375" uid: a4b7482b-521f-4efc-8326-f7e61d8bcf9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbc26} kind: Pod name: sequence-graph-fbc26-59cf79489b-jkc4g namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 441214a3-86a7-4c2c-b1d6-6aef7b931403 kind: Event lastTimestamp: "2026-07-07T14:35:46Z" message: Started container sequence-graph-fbc26 metadata: creationTimestamp: "2026-07-07T14:35:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:46Z" name: sequence-graph-fbc26-59cf79489b-jkc4g.18c008a29abf04dc namespace: kserve-ci-e2e-test resourceVersion: "22376" uid: 0cdba96e-1448-4292-8b36-c60dfea9d07d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbc26} kind: Pod name: sequence-graph-fbc26-59cf79489b-jkc4g namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 441214a3-86a7-4c2c-b1d6-6aef7b931403 kind: Event lastTimestamp: "2026-07-07T14:35:55Z" message: Stopping container sequence-graph-fbc26 metadata: creationTimestamp: "2026-07-07T14:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:55Z" name: sequence-graph-fbc26-59cf79489b-jkc4g.18c008a4b07264dd namespace: kserve-ci-e2e-test resourceVersion: "22480" uid: 3f396246-03ec-44b5-ad60-94b3c03255fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbc26} kind: Pod name: sequence-graph-fbc26-59cf79489b-jkc4g namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 441214a3-86a7-4c2c-b1d6-6aef7b931403 kind: Event lastTimestamp: "2026-07-07T14:36:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:36:23Z" name: sequence-graph-fbc26-59cf79489b-jkc4g.18c008a541d44c69 namespace: kserve-ci-e2e-test resourceVersion: "22869" uid: 4a04b264-feb4-41d1-a0af-1bf74f36a4c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-fbc26-59cf79489b namespace: kserve-ci-e2e-test resourceVersion: "22339" uid: 5790e9bc-9d21-48d6-b714-9b22634abc62 kind: Event lastTimestamp: "2026-07-07T14:35:45Z" message: 'Created pod: sequence-graph-fbc26-59cf79489b-jkc4g' metadata: creationTimestamp: "2026-07-07T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:45Z" name: sequence-graph-fbc26-59cf79489b.18c008a256d977a2 namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: 124fcb7f-b86e-44a7-aacf-4308d1bcf911 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "22337" uid: e9cab1e1-7a5c-4362-b79f-683b2ab173c9 kind: Event lastTimestamp: "2026-07-07T14:35:45Z" message: Scaled up replica set sequence-graph-fbc26-59cf79489b from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:45Z" name: sequence-graph-fbc26.18c008a255fe0510 namespace: kserve-ci-e2e-test resourceVersion: "22340" uid: 09a5269f-2dc8-4e61-8840-81f35a3f06e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "22355" uid: 6af11bbb-ced6-4c0f-8f6d-048fab8d7522 kind: Event lastTimestamp: "2026-07-07T14:35:45Z" message: InferenceGraph [sequence-graph-fbc26] is Ready metadata: creationTimestamp: "2026-07-07T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:45Z" name: sequence-graph-fbc26.18c008a259dd9203 namespace: kserve-ci-e2e-test resourceVersion: "22357" uid: 6d63e8f3-5e89-4258-85c7-380ac1ae79fc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:45:21.312074Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6e67f-fcb647484-fllj6 namespace: kserve-ci-e2e-test resourceVersion: "27272" uid: d20185bb-2a2c-4f60-8fbe-90e80732d150 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-6e67f-fcb647484-fllj6 to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:45:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:45:21Z" name: splitter-graph-6e67f-fcb647484-fllj6.18c00928579ed455 namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: f3fce7c4-0c76-47ae-b224-bb4136bbde0b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:21Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6e67f-fcb647484-fllj6 namespace: kserve-ci-e2e-test resourceVersion: "27274" uid: d20185bb-2a2c-4f60-8fbe-90e80732d150 kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-6e67f-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:21Z" name: splitter-graph-6e67f-fcb647484-fllj6.18c0092860741b6e namespace: kserve-ci-e2e-test resourceVersion: "27291" uid: ff5373ce-40b8-4fb0-a709-20d5529db28a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:22Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6e67f-fcb647484-fllj6 namespace: kserve-ci-e2e-test resourceVersion: "27281" uid: d20185bb-2a2c-4f60-8fbe-90e80732d150 kind: Event lastTimestamp: "2026-07-07T14:45:22Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:45:22Z" name: splitter-graph-6e67f-fcb647484-fllj6.18c00928954d3087 namespace: kserve-ci-e2e-test resourceVersion: "27300" uid: baf916d6-6469-4da0-b545-d96b4e4654d4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e67f} kind: Pod name: splitter-graph-6e67f-fcb647484-fllj6 namespace: kserve-ci-e2e-test resourceVersion: "27274" uid: d20185bb-2a2c-4f60-8fbe-90e80732d150 kind: Event lastTimestamp: "2026-07-07T14:45:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:22Z" name: splitter-graph-6e67f-fcb647484-fllj6.18c0092896b17e68 namespace: kserve-ci-e2e-test resourceVersion: "27302" uid: 70f2c1b0-cfa1-43ec-a754-d29e9d0702e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e67f} kind: Pod name: splitter-graph-6e67f-fcb647484-fllj6 namespace: kserve-ci-e2e-test resourceVersion: "27274" uid: d20185bb-2a2c-4f60-8fbe-90e80732d150 kind: Event lastTimestamp: "2026-07-07T14:45:22Z" message: 'Created container: splitter-graph-6e67f' metadata: creationTimestamp: "2026-07-07T14:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:22Z" name: splitter-graph-6e67f-fcb647484-fllj6.18c0092899cf74b7 namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: a0bfdd73-be29-4f51-ac49-fbb0c403ca6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e67f} kind: Pod name: splitter-graph-6e67f-fcb647484-fllj6 namespace: kserve-ci-e2e-test resourceVersion: "27274" uid: d20185bb-2a2c-4f60-8fbe-90e80732d150 kind: Event lastTimestamp: "2026-07-07T14:45:22Z" message: Started container splitter-graph-6e67f metadata: creationTimestamp: "2026-07-07T14:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:22Z" name: splitter-graph-6e67f-fcb647484-fllj6.18c009289a45137e namespace: kserve-ci-e2e-test resourceVersion: "27305" uid: 31bf1b6b-1b1d-416d-a0fe-b0af2643e469 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e67f} kind: Pod name: splitter-graph-6e67f-fcb647484-fllj6 namespace: kserve-ci-e2e-test resourceVersion: "27274" uid: d20185bb-2a2c-4f60-8fbe-90e80732d150 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: Stopping container splitter-graph-6e67f metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:31Z" name: splitter-graph-6e67f-fcb647484-fllj6.18c0092ab0f67b68 namespace: kserve-ci-e2e-test resourceVersion: "27398" uid: f28b1312-ee9f-483d-a966-40c4f360ce08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e67f} kind: Pod name: splitter-graph-6e67f-fcb647484-fllj6 namespace: kserve-ci-e2e-test resourceVersion: "27274" uid: d20185bb-2a2c-4f60-8fbe-90e80732d150 kind: Event lastTimestamp: "2026-07-07T14:45:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:58Z" name: splitter-graph-6e67f-fcb647484-fllj6.18c0092b2d5474fd namespace: kserve-ci-e2e-test resourceVersion: "27751" uid: 91e011ce-9970-403d-92cc-b8d09cdfffab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-6e67f-fcb647484 namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: ec8df978-1f5c-4c54-9ca2-5e2a0672630f kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: 'Created pod: splitter-graph-6e67f-fcb647484-fllj6' metadata: creationTimestamp: "2026-07-07T14:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:21Z" name: splitter-graph-6e67f-fcb647484.18c0092857329958 namespace: kserve-ci-e2e-test resourceVersion: "27273" uid: 32621ab4-abba-4fe0-9c4c-4d2841ec9587 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-6e67f namespace: kserve-ci-e2e-test resourceVersion: "27264" uid: 7935bcba-1788-4c18-8381-255fb81b9bcd kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: Scaled up replica set splitter-graph-6e67f-fcb647484 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:21Z" name: splitter-graph-6e67f.18c009285662c315 namespace: kserve-ci-e2e-test resourceVersion: "27269" uid: 4c80036d-5024-4af9-aecc-0709bff8b8cd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6e67f namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: c27fb00e-ebca-4fb4-8fe0-568182234ee7 kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: InferenceGraph [splitter-graph-6e67f] is Ready metadata: creationTimestamp: "2026-07-07T14:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:21Z" name: splitter-graph-6e67f.18c009285be2c9d9 namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: b7349ce4-659a-42fd-a451-d709d54d73f2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6e67f namespace: kserve-ci-e2e-test resourceVersion: "27259" uid: c27fb00e-ebca-4fb4-8fe0-568182234ee7 kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: 'Failed to update status for InferenceGraph "splitter-graph-6e67f": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-6e67f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:21Z" name: splitter-graph-6e67f.18c009285e5fe271 namespace: kserve-ci-e2e-test resourceVersion: "27288" uid: 708f5a3a-bf2c-49b6-b41c-d805d2c2b502 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6e67f namespace: kserve-ci-e2e-test resourceVersion: "27259" uid: c27fb00e-ebca-4fb4-8fe0-568182234ee7 kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-6e67f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:21Z" name: splitter-graph-6e67f.18c009285e6022e4 namespace: kserve-ci-e2e-test resourceVersion: "27289" uid: 9e321822-2b57-45bb-a18c-f9cd8b181e3b reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:46:31.602186Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7beb0-6cc4b5b5bc-brf96 namespace: kserve-ci-e2e-test resourceVersion: "28106" uid: f4d40633-686a-4b6b-a8fa-b4cabeff4f57 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-7beb0-6cc4b5b5bc-brf96 to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:46:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:46:31Z" name: splitter-graph-7beb0-6cc4b5b5bc-brf96.18c00938b53cd80f namespace: kserve-ci-e2e-test resourceVersion: "28111" uid: a437c007-7eb1-48f4-ba46-6af769c32e82 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:31Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7beb0-6cc4b5b5bc-brf96 namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: f4d40633-686a-4b6b-a8fa-b4cabeff4f57 kind: Event lastTimestamp: "2026-07-07T14:46:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-7beb0-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:31Z" name: splitter-graph-7beb0-6cc4b5b5bc-brf96.18c00938bcc9ff2b namespace: kserve-ci-e2e-test resourceVersion: "28123" uid: f573addd-cd9a-4971-bb5a-38719feab2e8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:32Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7beb0-6cc4b5b5bc-brf96 namespace: kserve-ci-e2e-test resourceVersion: "28115" uid: f4d40633-686a-4b6b-a8fa-b4cabeff4f57 kind: Event lastTimestamp: "2026-07-07T14:46:32Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:46:32Z" name: splitter-graph-7beb0-6cc4b5b5bc-brf96.18c00938f29dec4f namespace: kserve-ci-e2e-test resourceVersion: "28132" uid: d6da5afc-ad2e-4d44-bd2d-4384b50ff6e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7beb0} kind: Pod name: splitter-graph-7beb0-6cc4b5b5bc-brf96 namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: f4d40633-686a-4b6b-a8fa-b4cabeff4f57 kind: Event lastTimestamp: "2026-07-07T14:46:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:32Z" name: splitter-graph-7beb0-6cc4b5b5bc-brf96.18c00938f3de1bdf namespace: kserve-ci-e2e-test resourceVersion: "28134" uid: 468a0af6-558b-4163-afd7-65cf630d014d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7beb0} kind: Pod name: splitter-graph-7beb0-6cc4b5b5bc-brf96 namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: f4d40633-686a-4b6b-a8fa-b4cabeff4f57 kind: Event lastTimestamp: "2026-07-07T14:46:32Z" message: 'Created container: splitter-graph-7beb0' metadata: creationTimestamp: "2026-07-07T14:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:32Z" name: splitter-graph-7beb0-6cc4b5b5bc-brf96.18c00938f6c38e07 namespace: kserve-ci-e2e-test resourceVersion: "28135" uid: f2f26727-39c9-46bc-9677-fc48fc8608c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7beb0} kind: Pod name: splitter-graph-7beb0-6cc4b5b5bc-brf96 namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: f4d40633-686a-4b6b-a8fa-b4cabeff4f57 kind: Event lastTimestamp: "2026-07-07T14:46:32Z" message: Started container splitter-graph-7beb0 metadata: creationTimestamp: "2026-07-07T14:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:32Z" name: splitter-graph-7beb0-6cc4b5b5bc-brf96.18c00938f77fe381 namespace: kserve-ci-e2e-test resourceVersion: "28136" uid: 6ba8d60a-e018-4aa8-b711-f3d3a0e3885f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7beb0} kind: Pod name: splitter-graph-7beb0-6cc4b5b5bc-brf96 namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: f4d40633-686a-4b6b-a8fa-b4cabeff4f57 kind: Event lastTimestamp: "2026-07-07T14:54:46Z" message: Stopping container splitter-graph-7beb0 metadata: creationTimestamp: "2026-07-07T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:54:46Z" name: splitter-graph-7beb0-6cc4b5b5bc-brf96.18c009abe3fe5087 namespace: kserve-ci-e2e-test resourceVersion: "31555" uid: d1b37db2-6e7b-4301-97d5-7beb9bfbb7ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:54:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7beb0} kind: Pod name: splitter-graph-7beb0-6cc4b5b5bc-brf96 namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: f4d40633-686a-4b6b-a8fa-b4cabeff4f57 kind: Event lastTimestamp: "2026-07-07T14:55:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:55:13Z" name: splitter-graph-7beb0-6cc4b5b5bc-brf96.18c009ac77b19f7d namespace: kserve-ci-e2e-test resourceVersion: "31793" uid: 93a63889-c75f-496d-8ae8-026e45028af3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-7beb0-6cc4b5b5bc namespace: kserve-ci-e2e-test resourceVersion: "28102" uid: 0a6d3510-4c4c-4eea-a437-6d2ad696d256 kind: Event lastTimestamp: "2026-07-07T14:46:31Z" message: 'Created pod: splitter-graph-7beb0-6cc4b5b5bc-brf96' metadata: creationTimestamp: "2026-07-07T14:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:31Z" name: splitter-graph-7beb0-6cc4b5b5bc.18c00938b4c612ae namespace: kserve-ci-e2e-test resourceVersion: "28107" uid: 1221aa21-602a-482b-9e0e-25785ff75d9e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "28098" uid: a948999d-7975-4bec-950d-1e851c2030c5 kind: Event lastTimestamp: "2026-07-07T14:46:31Z" message: Scaled up replica set splitter-graph-7beb0-6cc4b5b5bc from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:31Z" name: splitter-graph-7beb0.18c00938b41a9713 namespace: kserve-ci-e2e-test resourceVersion: "28104" uid: a194c54a-a9ec-4442-8a0f-be1344732eff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "28118" uid: 3ea23305-a696-42d7-8b41-64a9ec6d18f7 kind: Event lastTimestamp: "2026-07-07T14:46:31Z" message: InferenceGraph [splitter-graph-7beb0] is Ready metadata: creationTimestamp: "2026-07-07T14:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:46:31Z" name: splitter-graph-7beb0.18c00938b967f3d6 namespace: kserve-ci-e2e-test resourceVersion: "28121" uid: 14fffa08-87f1-4d65-a1e4-fa1dcff70cf5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "28101" uid: 9b71ac54-3dee-4c80-ae0a-9c89d48fc852 kind: Event lastTimestamp: "2026-07-07T14:46:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:46Z" name: splitter-graph-7beb0.18c0093c3290dadc namespace: kserve-ci-e2e-test resourceVersion: "28242" uid: 817ab0bf-fa2f-4669-a896-382cc1c1fefd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "28101" uid: 9b71ac54-3dee-4c80-ae0a-9c89d48fc852 kind: Event lastTimestamp: "2026-07-07T14:46:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:46Z" name: splitter-graph-7beb0.18c0093c32ffaa5e namespace: kserve-ci-e2e-test resourceVersion: "28245" uid: 677a4087-960a-4a2c-9c01-ca1d50ab6be6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:47:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "28243" uid: 9b71ac54-3dee-4c80-ae0a-9c89d48fc852 kind: Event lastTimestamp: "2026-07-07T14:47:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:47:01Z" name: splitter-graph-7beb0.18c0093fb197402b namespace: kserve-ci-e2e-test resourceVersion: "28354" uid: 3393f618-2759-48e5-8b5f-5c23454c917d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:47:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "28243" uid: 9b71ac54-3dee-4c80-ae0a-9c89d48fc852 kind: Event lastTimestamp: "2026-07-07T14:47:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:47:01Z" name: splitter-graph-7beb0.18c0093fb2056050 namespace: kserve-ci-e2e-test resourceVersion: "28356" uid: 99b4067e-ecb7-49a3-a157-739929cc7ae1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:33:56.065692Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-03875-predictor-567dd775dc-h9f8v to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c00888cbc0c7ca namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 7211f64e-2cc8-4fca-9fa6-b8ac33ab3db9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c00888e56bd9c4 namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: b89f6418-d192-434d-8377-e01b581db10c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c00888e7041f72 namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: ab3e176c-4900-435e-b87e-5ec62e065d73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c00888ea1f5037 namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: 3efd3281-53ae-4371-8fed-5b5c1b2c6466 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c00888ea982510 namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 557a89f3-f70b-4ea6-bd2a-cfbf164dfdbc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c00888eaa2aad0 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 1d0b4f42-906e-4f27-99fc-ec9b7566f729 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c00888ed80fa79 namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 401c5e70-1100-48fa-83e1-d34cba6ad184 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c00888edf84e89 namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 8db07e34-b06f-4ba1-9bee-632e64303b8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:34:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:32Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c00888f982cb19 namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: f34c8d67-b3bc-4216-a031-b681570b98cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:06Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c008992664b9dc namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: e6f39b0c-910a-4fa6-b51e-9eafe9ae5657 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:06Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c0089926652b03 namespace: kserve-ci-e2e-test resourceVersion: "21903" uid: b74e97df-6179-421e-9654-4bbce8b03396 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03875-predictor-567dd775dc-h9f8v namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 8b4c350c-b103-46a3-879c-cd92be1dd15b kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: success-200-isvc-03875-predictor-567dd775dc-h9f8v.18c0089981d64942 namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: eb4555f9-5bbe-425f-9a2f-c453a6af64d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-03875-predictor-567dd775dc namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: aae80b18-69e8-48b4-9712-2380c17b9950 kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: 'Created pod: success-200-isvc-03875-predictor-567dd775dc-h9f8v' metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875-predictor-567dd775dc.18c00888cb0fe4fb namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: e17b41ca-0039-4581-adb3-34a2d7558cfe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 42f748b0-2aac-4fb1-8fa9-7b460c2ad3ee kind: Event lastTimestamp: "2026-07-07T14:33:55Z" message: Scaled up replica set success-200-isvc-03875-predictor-567dd775dc from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:33:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:55Z" name: success-200-isvc-03875-predictor.18c00888c73b8336 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 5dc467d4-5296-4948-b9ac-7f711cb42813 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:34:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 71a76619-d520-446f-8459-201303aa9f77 kind: Event lastTimestamp: "2026-07-07T14:34:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:41Z" name: success-200-isvc-03875-predictor.18c0088c461e876e namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: d9092b8c-9b48-46fc-8950-92578f17c4cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:34:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 71a76619-d520-446f-8459-201303aa9f77 kind: Event lastTimestamp: "2026-07-07T14:34:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:41Z" name: success-200-isvc-03875-predictor.18c0088c466afe22 namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: ffda6b1a-7317-4352-8629-95e24b60e502 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 71a76619-d520-446f-8459-201303aa9f77 kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:56Z" name: success-200-isvc-03875-predictor.18c00896c32ef96e namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 09ee3b90-72b5-4471-b92d-0e220affbc7e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03875-predictor namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 71a76619-d520-446f-8459-201303aa9f77 kind: Event lastTimestamp: "2026-07-07T14:34:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:34:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:56Z" name: success-200-isvc-03875-predictor.18c00896c3a4b34e namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: bbc8472d-2c83-426e-b020-2bceded7f5ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:33:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-03875 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: ba95b081-f2b8-4f98-aa9e-95fe20f8d430 kind: Event lastTimestamp: "2026-07-07T14:34:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:33:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:34:42Z" name: success-200-isvc-03875.18c00888c740aa6f namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: dd3b9195-ea89-4bd4-ad46-1fc2cb4bb5e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-03875 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: ba95b081-f2b8-4f98-aa9e-95fe20f8d430 kind: Event lastTimestamp: "2026-07-07T14:33:55Z" message: 'Failed to update status for InferenceService "success-200-isvc-03875": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-03875": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:33:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:55Z" name: success-200-isvc-03875.18c00888c7b23f33 namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: 72ee886b-4e9c-4746-bebb-e95013a4b18c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-03875 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: ba95b081-f2b8-4f98-aa9e-95fe20f8d430 kind: Event lastTimestamp: "2026-07-07T14:33:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-03875": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-03875.18c00888c7b2867f namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 5c733c42-3ec3-456c-ba0c-3dc3c85d88eb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-03875 namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: ba95b081-f2b8-4f98-aa9e-95fe20f8d430 kind: Event lastTimestamp: "2026-07-07T14:34:42Z" message: InferenceService [success-200-isvc-03875] is Ready metadata: creationTimestamp: "2026-07-07T14:34:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:34:42Z" name: success-200-isvc-03875.18c00893b4dd5f18 namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: 769d68d6-7356-4503-95e2-de7f63949338 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:45:10.972413Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27077" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-36f00-predictor-869b59995f-knmkk to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:45:10Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c00925ef541488 namespace: kserve-ci-e2e-test resourceVersion: "27083" uid: 965a1043-49ed-4b60-8345-2b82d2aebd6c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T14:45:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-36f00-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:11Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c00925fbce3340 namespace: kserve-ci-e2e-test resourceVersion: "27136" uid: 7d05abbf-45ee-4d7e-92ef-609a2aafcd0e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:45:12Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c009262cd459f0 namespace: kserve-ci-e2e-test resourceVersion: "27151" uid: 81f82f57-7d48-4902-8497-6d77922dda04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c009262e457ff5 namespace: kserve-ci-e2e-test resourceVersion: "27153" uid: 56460f3a-4495-49f8-a5bf-842ef68daed6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c0092631bcb47b namespace: kserve-ci-e2e-test resourceVersion: "27154" uid: ef357480-779f-40e6-95ed-8b828cc5a818 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c0092632351507 namespace: kserve-ci-e2e-test resourceVersion: "27156" uid: 7b319969-ad5a-4c4d-a027-bac7e8a231bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c00926323f5821 namespace: kserve-ci-e2e-test resourceVersion: "27157" uid: c8c14cb6-f5dc-49fe-a72d-1bb00c7e37e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c00926354b034c namespace: kserve-ci-e2e-test resourceVersion: "27160" uid: c65bd23c-9f04-4266-88a5-04169daf6e9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T14:45:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:45:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:12Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c0092636041ff4 namespace: kserve-ci-e2e-test resourceVersion: "27161" uid: 3557276f-48a9-4526-bee1-8f0a15d0132d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T14:45:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:49Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c009268281f895 namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: 555809ab-98cd-42da-bc0d-5c05ea76fce6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T15:02:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:02:30Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c00a17eec83ac7 namespace: kserve-ci-e2e-test resourceVersion: "34777" uid: d9e8eeb6-5e18-4d63-95df-e05c36be32d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-36f00-predictor-869b59995f-knmkk namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 3c525a2c-686b-4c1c-afd0-e826d769d80a kind: Event lastTimestamp: "2026-07-07T15:02:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:02:30Z" name: success-200-isvc-36f00-predictor-869b59995f-knmkk.18c00a17eec88fb9 namespace: kserve-ci-e2e-test resourceVersion: "34778" uid: ad36fa93-1864-4c24-aa11-55ea89a330c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-36f00-predictor-869b59995f namespace: kserve-ci-e2e-test resourceVersion: "27061" uid: 5b0ee643-d568-4608-9d0c-ace0c3dafcf0 kind: Event lastTimestamp: "2026-07-07T14:45:10Z" message: 'Created pod: success-200-isvc-36f00-predictor-869b59995f-knmkk' metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:10Z" name: success-200-isvc-36f00-predictor-869b59995f.18c00925eee8860c namespace: kserve-ci-e2e-test resourceVersion: "27079" uid: 158ffc2c-4750-4a3b-967d-b22cf3b144b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27053" uid: 76466a22-b9f4-4f72-8991-8ac3bb1847fe kind: Event lastTimestamp: "2026-07-07T14:45:10Z" message: Scaled up replica set success-200-isvc-36f00-predictor-869b59995f from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:10Z" name: success-200-isvc-36f00-predictor.18c00925ece076f1 namespace: kserve-ci-e2e-test resourceVersion: "27065" uid: 2ab997db-f440-4042-90c9-f5ee7fa9f1cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27059" uid: 2bba7a1c-04d7-4607-aa02-b8c3b922acc6 kind: Event lastTimestamp: "2026-07-07T14:45:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:55Z" name: success-200-isvc-36f00-predictor.18c009296b5ea3b9 namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: d4a38160-f6f8-44da-a57b-284c8c705e3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27059" uid: 2bba7a1c-04d7-4607-aa02-b8c3b922acc6 kind: Event lastTimestamp: "2026-07-07T14:45:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:55Z" name: success-200-isvc-36f00-predictor.18c009296bc0e021 namespace: kserve-ci-e2e-test resourceVersion: "27730" uid: 7a2b062c-ed67-4eb7-bff8-7710b186f9dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:46:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27335" uid: 2bba7a1c-04d7-4607-aa02-b8c3b922acc6 kind: Event lastTimestamp: "2026-07-07T14:46:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:26Z" name: success-200-isvc-36f00-predictor.18c00933e89259ab namespace: kserve-ci-e2e-test resourceVersion: "28048" uid: c9caa2c9-0181-441d-afb2-59dcef712d20 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:46:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-36f00-predictor namespace: kserve-ci-e2e-test resourceVersion: "27335" uid: 2bba7a1c-04d7-4607-aa02-b8c3b922acc6 kind: Event lastTimestamp: "2026-07-07T14:46:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:26Z" name: success-200-isvc-36f00-predictor.18c00933e8ee226c namespace: kserve-ci-e2e-test resourceVersion: "28049" uid: 1ba3bc6d-f35d-4bcd-b545-b227d82e22a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: 79171a12-c3ab-496f-ac50-b5c21567755b kind: Event lastTimestamp: "2026-07-07T14:45:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:59Z" name: success-200-isvc-36f00.18c00925ecf3bcce namespace: kserve-ci-e2e-test resourceVersion: "27776" uid: ad5dd049-9291-4279-a0b1-8fa2af41fe0d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: 79171a12-c3ab-496f-ac50-b5c21567755b kind: Event lastTimestamp: "2026-07-07T14:45:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-36f00": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-36f00": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:10Z" name: success-200-isvc-36f00.18c00925edb4d767 namespace: kserve-ci-e2e-test resourceVersion: "27072" uid: 064db47e-fa6d-4abd-9c32-aa1fc46270e9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: 79171a12-c3ab-496f-ac50-b5c21567755b kind: Event lastTimestamp: "2026-07-07T14:45:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-36f00": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:10Z" name: success-200-isvc-36f00.18c00925edb53142 namespace: kserve-ci-e2e-test resourceVersion: "27075" uid: bad59af6-0bd9-4a5b-a0cf-bc7930536435 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27774" uid: 79171a12-c3ab-496f-ac50-b5c21567755b kind: Event lastTimestamp: "2026-07-07T14:45:59Z" message: InferenceService [success-200-isvc-36f00] is Ready metadata: creationTimestamp: "2026-07-07T14:45:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:59Z" name: success-200-isvc-36f00.18c009313da53b6e namespace: kserve-ci-e2e-test resourceVersion: "27775" uid: 1df2815e-6b49-4d76-9dba-2d3e22e9933e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:24:31.002369Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-520df-predictor-86849fb99-w5khm to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:24:31Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c008053b5af3f8 namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: 16f1c11d-0f1c-4d7b-b0ef-9c50d80f91b3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:24:31Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c0080554f7f6fc namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 75476e54-11d3-47a1-a28d-189c437f9d54 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c0080556818acf namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 6d892803-9ec8-4900-8162-2fdc9c1f1797 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c008055994f024 namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 473f97c9-b173-4864-8180-6af2d8f71e79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c008055a0ad4cc namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: d9f3711a-5fdc-4113-804f-c3cbe66aa74d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c008055a15cbbb namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 3cf5d09a-b061-43eb-83ad-06a7b6f8ec8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c008055cfd0d89 namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: 845db664-7841-4c84-8f7d-f7b4af8a45a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:24:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:31Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c008055d74e201 namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: ec618474-a982-46ea-a503-01f072055229 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-07T14:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:33:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:57Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c00805633906f5 namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: 9ee8ceb2-55bb-46e4-b109-204c4266e0e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c00888cb13fe2d namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: da13b3fe-9a86-4848-809c-66af33b22bf8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:33:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:56Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c00888cb1531c5 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: be8c79cc-c365-4289-9277-48d4bd9d567c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-520df-predictor-86849fb99-w5khm namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 12825a42-a286-48f8-b8cf-493a38b1b7b0 kind: Event lastTimestamp: "2026-07-07T14:33:57Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:57Z" name: success-200-isvc-520df-predictor-86849fb99-w5khm.18c008892bbdcf79 namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: a4158e97-c73a-444d-9125-2f940b5d58b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-520df-predictor-86849fb99 namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: aee0a17e-49f7-4633-a4a5-f757d693461f kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: 'Created pod: success-200-isvc-520df-predictor-86849fb99-w5khm' metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:30Z" name: success-200-isvc-520df-predictor-86849fb99.18c008053ad6a7dd namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 6cae2195-48e8-427f-b94a-cf473e215630 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: 30cf3ca3-a310-4944-887e-4d8357c82ada kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Scaled up replica set success-200-isvc-520df-predictor-86849fb99 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:30Z" name: success-200-isvc-520df-predictor.18c0080538c8e201 namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: b72d8758-90c3-4518-8b35-93cae7e0771b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:24:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: 410cf909-5b8e-4a71-929f-15ce12677c8c kind: Event lastTimestamp: "2026-07-07T14:25:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:46Z" name: success-200-isvc-520df-predictor.18c00808b7b52961 namespace: kserve-ci-e2e-test resourceVersion: "17526" uid: 267cfa22-954d-4278-90be-d80417e00126 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:24:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: 410cf909-5b8e-4a71-929f-15ce12677c8c kind: Event lastTimestamp: "2026-07-07T14:25:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:46Z" name: success-200-isvc-520df-predictor.18c00808b80ce3f3 namespace: kserve-ci-e2e-test resourceVersion: "17527" uid: 70a60b57-1807-4ac4-866b-884fa6b8734c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16773" uid: 410cf909-5b8e-4a71-929f-15ce12677c8c kind: Event lastTimestamp: "2026-07-07T14:25:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:00Z" name: success-200-isvc-520df-predictor.18c0080c36bae1bd namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: a9ed2f30-f2c6-4f36-a3df-634c3800edd4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-520df-predictor namespace: kserve-ci-e2e-test resourceVersion: "16773" uid: 410cf909-5b8e-4a71-929f-15ce12677c8c kind: Event lastTimestamp: "2026-07-07T14:25:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:00Z" name: success-200-isvc-520df-predictor.18c0080c3732d1a5 namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 6d988eba-ac8a-44ff-b57c-15648a8d2402 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-520df namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 7b7ee89d-dba0-450a-ada1-ad5cf9a2aae6 kind: Event lastTimestamp: "2026-07-07T14:25:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:27Z" name: success-200-isvc-520df.18c008053964acae namespace: kserve-ci-e2e-test resourceVersion: "17172" uid: 27c083ff-2322-4c49-9643-045d84eb94e7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-520df namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 7b7ee89d-dba0-450a-ada1-ad5cf9a2aae6 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: 'Failed to update status for InferenceService "success-200-isvc-520df": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-520df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:30Z" name: success-200-isvc-520df.18c0080539ffb96e namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 0a227468-bf85-4c31-bf58-4bc7c392c6b1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-520df namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 7b7ee89d-dba0-450a-ada1-ad5cf9a2aae6 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-520df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:30Z" name: success-200-isvc-520df.18c008053a001922 namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 4e04b83c-3f32-48d5-b613-e20629d0cb44 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-520df namespace: kserve-ci-e2e-test resourceVersion: "17169" uid: 7b7ee89d-dba0-450a-ada1-ad5cf9a2aae6 kind: Event lastTimestamp: "2026-07-07T14:25:27Z" message: InferenceService [success-200-isvc-520df] is Ready metadata: creationTimestamp: "2026-07-07T14:25:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:27Z" name: success-200-isvc-520df.18c00812759d9ece namespace: kserve-ci-e2e-test resourceVersion: "17171" uid: aeb2f8ec-12ed-4d73-af55-195a86ceda9c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:44:21.283490Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:44:21Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091a5da35e7f namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: d097893d-b7ef-40c8-9058-bca05b5a6063 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6e67f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091a686a38fd namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: c0dc5e52-53e4-4ec2-9b06-3dae87085802 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:44:22Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:44:22Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091a9b1cef46 namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 3415612a-6798-436c-b70e-8eaf05c04def reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:44:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:22Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091a9ca71725 namespace: kserve-ci-e2e-test resourceVersion: "26624" uid: c26bd356-f744-44be-8b22-babf631dc33c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:44:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:22Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091aa02d3fd7 namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: aa60d7ba-cc40-420f-a871-e94b54ca9748 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:44:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:22Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091aa0a77179 namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 09294b70-4c83-43b3-bfd4-26598bf9cc0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:44:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:22Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091aa0b2323f namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: cbcd8fd1-b23f-4255-a6ce-74244df6464f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:44:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:22Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091aa532e1ee namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: ae28d6e2-8ffb-4ef3-93f8-7e0751d6940e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:44:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:22Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091aa5adfcf8 namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 981f9849-08a6-4661-a0fc-a6ff28a73603 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:44:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:45:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:00Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0091b0dfa369b namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 71254ebc-12fb-4dd1-9361-b253bdae23d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:31Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0092ab79ab98b namespace: kserve-ci-e2e-test resourceVersion: "27429" uid: 7e52d6b8-3052-4c7e-8eff-e7101fe3b6d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 7f271e4c-1439-419a-8b46-04b1c7b34595 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:31Z" name: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj.18c0092ab79c5901 namespace: kserve-ci-e2e-test resourceVersion: "27431" uid: 1346026e-c52f-4dbf-ba3d-3f2cca894001 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6e67f-predictor-787f79cf5c namespace: kserve-ci-e2e-test resourceVersion: "26520" uid: 34d414f9-c1d9-4930-a3f5-82a73fef9c93 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: 'Created pod: success-200-isvc-6e67f-predictor-787f79cf5c-wnvpj' metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:44:21Z" name: success-200-isvc-6e67f-predictor-787f79cf5c.18c0091a5cfb5ae2 namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: eacb67f2-c098-4532-8c09-74d5627c74ce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: 604b8f04-8ff5-4c04-90ff-50f133bd19bc kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Scaled up replica set success-200-isvc-6e67f-predictor-787f79cf5c from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:44:21Z" name: success-200-isvc-6e67f-predictor.18c0091a599148a9 namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: 13faaf49-4bc9-4b35-96e0-f7ee3ef7aabb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:44:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: e3b41caf-7cc5-434f-bf26-2fd5561c9300 kind: Event lastTimestamp: "2026-07-07T14:44:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:44:51Z" name: success-200-isvc-6e67f-predictor.18c0091dd7ef2401 namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: 1be6d904-7332-426c-8a42-df02af2de18e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:44:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: e3b41caf-7cc5-434f-bf26-2fd5561c9300 kind: Event lastTimestamp: "2026-07-07T14:44:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:44:51Z" name: success-200-isvc-6e67f-predictor.18c0091dd8531fb0 namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 7e5e7602-1f44-4495-aeaf-f60dafa7a18e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:45:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: e3b41caf-7cc5-434f-bf26-2fd5561c9300 kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:21Z" name: success-200-isvc-6e67f-predictor.18c00924d65edc38 namespace: kserve-ci-e2e-test resourceVersion: "27266" uid: 83958b0e-d34e-4bed-bb1e-a748993c58dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:45:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e67f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: e3b41caf-7cc5-434f-bf26-2fd5561c9300 kind: Event lastTimestamp: "2026-07-07T14:45:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:21Z" name: success-200-isvc-6e67f-predictor.18c00924d6da77d6 namespace: kserve-ci-e2e-test resourceVersion: "27271" uid: 8870b8c6-35d9-4d5c-ba08-dd58699962d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e67f namespace: kserve-ci-e2e-test resourceVersion: "26509" uid: 75a9081b-2305-469f-8bf0-b285ff549e46 kind: Event lastTimestamp: "2026-07-07T14:45:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:10Z" name: success-200-isvc-6e67f.18c0091a598c52af namespace: kserve-ci-e2e-test resourceVersion: "27020" uid: 295498dd-99d3-490e-b904-a70bc0f75c20 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e67f namespace: kserve-ci-e2e-test resourceVersion: "26509" uid: 75a9081b-2305-469f-8bf0-b285ff549e46 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-6e67f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6e67f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:44:21Z" name: success-200-isvc-6e67f.18c0091a59e3c71e namespace: kserve-ci-e2e-test resourceVersion: "26524" uid: bb6bd517-e8a8-43b9-a172-6d44d2721edd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e67f namespace: kserve-ci-e2e-test resourceVersion: "26509" uid: 75a9081b-2305-469f-8bf0-b285ff549e46 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6e67f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:44:21Z" name: success-200-isvc-6e67f.18c0091a59e40950 namespace: kserve-ci-e2e-test resourceVersion: "26525" uid: 29afaf2e-e09f-4974-8ea0-0827d13d01f1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e67f namespace: kserve-ci-e2e-test resourceVersion: "27017" uid: 75a9081b-2305-469f-8bf0-b285ff549e46 kind: Event lastTimestamp: "2026-07-07T14:45:10Z" message: InferenceService [success-200-isvc-6e67f] is Ready metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:10Z" name: success-200-isvc-6e67f.18c00925c9034fbf namespace: kserve-ci-e2e-test resourceVersion: "27019" uid: 7b691bd4-c66c-4e79-98db-451bc32ef813 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:45:31.536482Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27435" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:45:31Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092ab90ae4a7 namespace: kserve-ci-e2e-test resourceVersion: "27441" uid: af61621d-7d52-4eac-bc3c-fbd55c88bafe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7beb0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:31Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092ac57a2c88 namespace: kserve-ci-e2e-test resourceVersion: "27492" uid: 2bc4d2a9-cfdc-4939-a7d6-da4b04d9381a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27446" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:45:32Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092af687865e namespace: kserve-ci-e2e-test resourceVersion: "27507" uid: a0311cfa-2b28-4390-8f50-884cd605d436 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092af81eee58 namespace: kserve-ci-e2e-test resourceVersion: "27509" uid: 4823d2b7-dd0f-4ae8-9f34-3751a50a22f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092afb5acb27 namespace: kserve-ci-e2e-test resourceVersion: "27510" uid: 08e253aa-e746-4d54-8764-5f06407aab62 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092afbd00250 namespace: kserve-ci-e2e-test resourceVersion: "27511" uid: 95e4c6b3-e14f-4f24-ba63-55353d5d6248 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092afbda337d namespace: kserve-ci-e2e-test resourceVersion: "27513" uid: b52db61a-1c1c-4cba-9d91-f499adebdb16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092afecc08d4 namespace: kserve-ci-e2e-test resourceVersion: "27518" uid: 1253a3c3-46c6-492d-9ef3-3f39cf40a6d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:45:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:32Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092aff464fc4 namespace: kserve-ci-e2e-test resourceVersion: "27519" uid: 710ed886-8623-4416-a3ac-037f12001470 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:46:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:10Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c0092b6bd4a53c namespace: kserve-ci-e2e-test resourceVersion: "27864" uid: 74aa98fe-8465-4713-9c26-210953818648 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:54:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:54:46Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c009abee8a9bdd namespace: kserve-ci-e2e-test resourceVersion: "31574" uid: 68e3c7a1-7be5-43bd-92f2-21b4ac000aed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 245eaf0e-81d9-4381-a501-33bb7f9952ef kind: Event lastTimestamp: "2026-07-07T14:54:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:54:46Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6.18c009abee8b2f9b namespace: kserve-ci-e2e-test resourceVersion: "31576" uid: b5240525-427a-4870-9a61-3d567ebf6356 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7beb0-predictor-5d5b878bd9 namespace: kserve-ci-e2e-test resourceVersion: "27423" uid: ee306621-a364-4cb9-9d72-96bbdaf09361 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: 'Created pod: success-200-isvc-7beb0-predictor-5d5b878bd9-dbpz6' metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:31Z" name: success-200-isvc-7beb0-predictor-5d5b878bd9.18c0092ab89cc05b namespace: kserve-ci-e2e-test resourceVersion: "27437" uid: bc02d5f0-9607-4df2-836d-2c91d74af66f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27415" uid: 0fe521eb-bf46-4d6a-bfe4-c3c15a9812a6 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: Scaled up replica set success-200-isvc-7beb0-predictor-5d5b878bd9 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:45:31Z" name: success-200-isvc-7beb0-predictor.18c0092ab70215bd namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: b63b95ce-9850-40b1-af54-7df852255d88 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27422" uid: ed0d140b-c1f5-4a14-9f43-962e58ba6f57 kind: Event lastTimestamp: "2026-07-07T14:46:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:31Z" name: success-200-isvc-7beb0-predictor.18c0092e36274313 namespace: kserve-ci-e2e-test resourceVersion: "28099" uid: 62ea8483-02b3-4f7d-ae08-49087aa6bd1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27422" uid: ed0d140b-c1f5-4a14-9f43-962e58ba6f57 kind: Event lastTimestamp: "2026-07-07T14:46:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:31Z" name: success-200-isvc-7beb0-predictor.18c0092e3694a8be namespace: kserve-ci-e2e-test resourceVersion: "28103" uid: 4c3ff69d-7945-434c-b7db-14829cbba923 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27657" uid: ed0d140b-c1f5-4a14-9f43-962e58ba6f57 kind: Event lastTimestamp: "2026-07-07T14:46:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:01Z" name: success-200-isvc-7beb0-predictor.18c00931b545d8e2 namespace: kserve-ci-e2e-test resourceVersion: "27792" uid: 09a26a06-5ce8-417e-8f24-9ab369fb7fa5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7beb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "27657" uid: ed0d140b-c1f5-4a14-9f43-962e58ba6f57 kind: Event lastTimestamp: "2026-07-07T14:46:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:01Z" name: success-200-isvc-7beb0-predictor.18c00931b5ac60ba namespace: kserve-ci-e2e-test resourceVersion: "27794" uid: ddd9ffcc-89eb-460a-9796-925b01eb665d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "27403" uid: d78fbbd9-3999-45f4-b68f-1798f0d108e6 kind: Event lastTimestamp: "2026-07-07T14:46:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:46:20Z" name: success-200-isvc-7beb0.18c0092ab70afdf1 namespace: kserve-ci-e2e-test resourceVersion: "28007" uid: 2cbe8350-eb9f-4a47-aa4e-35c22abbb4a5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "27403" uid: d78fbbd9-3999-45f4-b68f-1798f0d108e6 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-7beb0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7beb0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:31Z" name: success-200-isvc-7beb0.18c0092ab7a1a6b5 namespace: kserve-ci-e2e-test resourceVersion: "27430" uid: 4b1ccdc8-0e86-4c99-bcaf-55a9f2d63468 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "27403" uid: d78fbbd9-3999-45f4-b68f-1798f0d108e6 kind: Event lastTimestamp: "2026-07-07T14:45:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7beb0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:45:31Z" name: success-200-isvc-7beb0.18c0092ab7a1ede6 namespace: kserve-ci-e2e-test resourceVersion: "27433" uid: 42d1f325-298a-4aa2-839b-cf8cd9b61ef9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7beb0 namespace: kserve-ci-e2e-test resourceVersion: "28005" uid: d78fbbd9-3999-45f4-b68f-1798f0d108e6 kind: Event lastTimestamp: "2026-07-07T14:46:20Z" message: InferenceService [success-200-isvc-7beb0] is Ready metadata: creationTimestamp: "2026-07-07T14:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:46:20Z" name: success-200-isvc-7beb0.18c00936273059ad namespace: kserve-ci-e2e-test resourceVersion: "28006" uid: f2d9f134-a6ad-4818-9521-65fb67b76f24 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:23:00.615230Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:23:00Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f02fdc8bec namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 86b623c1-c0b9-4a40-a716-26258a385b3d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a2dea-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:00Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f0387c431a namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: 6380f9ae-8b29-41c3-838c-19ef5a95d59a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Add eth0 [10.134.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:23:01Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f06d8ffdd1 namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: 18fb4211-ee58-4190-b61e-7d9b506d6ecb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:01Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-07T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:01Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f070753cbe namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: a3d5593f-5ec7-45ec-b4ee-0079aee0f7ca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:14Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" in 13.058s (13.058s including waiting). Image size: 1335611503 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:14Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f37acb5061 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 691aa241-a5cb-408a-b852-ca065cae3f2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:14Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f37ead3cdc namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 927c5026-067e-4706-9935-6342c5c4ff26 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:14Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f37f2599ff namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 8ffd964a-933c-457b-bc83-e6e10d84d16a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:14Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T14:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:14Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f37f339fa9 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 5d58e6b6-0f82-43a4-a584-2602c250b2f9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:17Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.685s (2.685s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T14:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:17Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f41f43ba8d namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 35086125-bc94-4a20-a0d6-b6d99215997c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:17Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f4222e70d5 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 26578941-71a6-4215-b887-16a70d13ae4f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:17Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f422a39ce4 namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: 12030db2-6b18-4a12-bf3b-b63059bd2b77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:23:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:23:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:55Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c007f48f9b0b99 namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: 42cc626f-350f-4594-af6f-0f034299dc37 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c008053a1b09d0 namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: b4d8123b-4b9a-4b1a-8f28-05826d98842d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 567dd16a-6cf4-4d8c-92e4-9f1ceec627ef kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt.18c008053a1bd395 namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: 551a83d1-9376-49a2-82db-6e4a1dfb7a5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a2dea-predictor-68b6fb989f namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: b85615ac-5f1c-4803-b571-7d3a4b8817da kind: Event lastTimestamp: "2026-07-07T14:23:00Z" message: 'Created pod: success-200-isvc-a2dea-predictor-68b6fb989f-wb6nt' metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:00Z" name: success-200-isvc-a2dea-predictor-68b6fb989f.18c007f02f5d86f2 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: ea34f8f9-44b5-4617-8e97-a9d02e17f90a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: cd7f966e-931b-41ff-a36e-e2e4841961e4 kind: Event lastTimestamp: "2026-07-07T14:23:00Z" message: Scaled up replica set success-200-isvc-a2dea-predictor-68b6fb989f from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:00Z" name: success-200-isvc-a2dea-predictor.18c007f02b648845 namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: 221c7436-2332-4ddf-a28a-6d74a4db8817 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:23:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 7b092a83-eab5-4bd0-8228-a50fe2820180 kind: Event lastTimestamp: "2026-07-07T14:23:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:45Z" name: success-200-isvc-a2dea-predictor.18c007f3b19ccd43 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: c1064850-6107-4038-a91f-75d7d7213bfd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:23:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 7b092a83-eab5-4bd0-8228-a50fe2820180 kind: Event lastTimestamp: "2026-07-07T14:23:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:45Z" name: success-200-isvc-a2dea-predictor.18c007f3b232efb2 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: 405e42de-801f-473e-84e7-034a30456e4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:23:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: 7b092a83-eab5-4bd0-8228-a50fe2820180 kind: Event lastTimestamp: "2026-07-07T14:24:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:15Z" name: success-200-isvc-a2dea-predictor.18c007f7310b15f5 namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: 70d94b01-cffd-429b-8250-d3c8fcce0dd9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:23:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a2dea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: 7b092a83-eab5-4bd0-8228-a50fe2820180 kind: Event lastTimestamp: "2026-07-07T14:24:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:15Z" name: success-200-isvc-a2dea-predictor.18c007f7315a4d4c namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: b0c81e42-d60d-4725-97e0-8b1c878a2490 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:23:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a2dea namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 585e835e-0b26-467d-88ea-59c9a2773703 kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:05Z" name: success-200-isvc-a2dea.18c007f0316bc284 namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: 6a5f936e-7413-4f1b-82cf-b6de479a5222 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a2dea namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 585e835e-0b26-467d-88ea-59c9a2773703 kind: Event lastTimestamp: "2026-07-07T14:24:05Z" message: InferenceService [success-200-isvc-a2dea] is Ready metadata: creationTimestamp: "2026-07-07T14:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:05Z" name: success-200-isvc-a2dea.18c007ff4b33a2b9 namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 5b723fa9-d22d-4205-b784-4709ac6a1601 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:25:31.114139Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:25:31Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c008133a4b9cc8 namespace: kserve-ci-e2e-test resourceVersion: "17264" uid: edb4938a-006f-4ebf-bb84-89e56528f905 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ab362-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:31Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c0081342853592 namespace: kserve-ci-e2e-test resourceVersion: "17304" uid: d5c67e7f-cd77-4004-8d6f-f825602651d2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:25:32Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:25:32Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c0081377ae0198 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 693399dd-28b9-4fbb-9b8c-6b3cbcb591c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:25:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:32Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c00813799a8a15 namespace: kserve-ci-e2e-test resourceVersion: "17359" uid: 7a9c77c3-56ca-497e-87c1-35695fc9f69b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:25:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:32Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c008137d0d65c4 namespace: kserve-ci-e2e-test resourceVersion: "17363" uid: ec251285-bc3c-4cf8-8078-afd4d73e0f89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:25:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:32Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c008137d845059 namespace: kserve-ci-e2e-test resourceVersion: "17364" uid: f761cd8e-42a1-4102-b225-ddbbf41c8ed0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:25:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:32Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c008137d8f1535 namespace: kserve-ci-e2e-test resourceVersion: "17365" uid: d7bcec70-fbec-4390-933c-782f3ef62f00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:25:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:32Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c0081380de0766 namespace: kserve-ci-e2e-test resourceVersion: "17366" uid: e6d392cf-e092-444f-a69b-b14bc026e482 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:25:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:32Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c00813815548ea namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: 9abf6f69-7394-48f2-a14b-2b5b6ecb5378 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:25:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:26:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:09Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c00813e15546c6 namespace: kserve-ci-e2e-test resourceVersion: "17720" uid: fabe226a-874b-449e-834c-92392aff9f53 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:45Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c008945c49dad7 namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: 938e9214-0e2f-40da-b24a-b545d2d7312c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45 namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 83654928-fe75-4138-90ed-3acf87dad1ae kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:45Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45.18c008945c4a77ed namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: 8282bc09-dcde-45bd-8f5a-7f5f48096513 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ab362-predictor-5c95c8c5d9 namespace: kserve-ci-e2e-test resourceVersion: "17237" uid: d9fd948c-48cf-4393-815b-fb03ea12b427 kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: 'Created pod: success-200-isvc-ab362-predictor-5c95c8c5d9-bdd45' metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:31Z" name: success-200-isvc-ab362-predictor-5c95c8c5d9.18c0081339de8f97 namespace: kserve-ci-e2e-test resourceVersion: "17261" uid: 75247e46-3a5a-4c14-914f-cac9e43bf58c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ab362-predictor namespace: kserve-ci-e2e-test resourceVersion: "17234" uid: 430ad3c8-fa6a-4b72-8dc9-9b105fcb92c0 kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: Scaled up replica set success-200-isvc-ab362-predictor-5c95c8c5d9 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:31Z" name: success-200-isvc-ab362-predictor.18c00813372dc4bb namespace: kserve-ci-e2e-test resourceVersion: "17243" uid: ce46cf6e-3fdf-482d-9157-6d8609930e98 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab362-predictor namespace: kserve-ci-e2e-test resourceVersion: "17236" uid: 7cb7d105-af47-4843-9428-0bd8030d0bfe kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:31Z" name: success-200-isvc-ab362-predictor.18c00816b6196ea9 namespace: kserve-ci-e2e-test resourceVersion: "17914" uid: 0a6eb433-fb33-4b49-95c9-015c5b155c21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab362-predictor namespace: kserve-ci-e2e-test resourceVersion: "17236" uid: 7cb7d105-af47-4843-9428-0bd8030d0bfe kind: Event lastTimestamp: "2026-07-07T14:26:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:31Z" name: success-200-isvc-ab362-predictor.18c00816b69e9ce4 namespace: kserve-ci-e2e-test resourceVersion: "17915" uid: 754bcdea-b2ba-40bd-a8b3-3640462ee594 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab362-predictor namespace: kserve-ci-e2e-test resourceVersion: "17528" uid: 7cb7d105-af47-4843-9428-0bd8030d0bfe kind: Event lastTimestamp: "2026-07-07T14:26:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:01Z" name: success-200-isvc-ab362-predictor.18c0081a354fa292 namespace: kserve-ci-e2e-test resourceVersion: "17653" uid: dffa0c02-c3a7-42a9-9474-d60145d505d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab362-predictor namespace: kserve-ci-e2e-test resourceVersion: "17528" uid: 7cb7d105-af47-4843-9428-0bd8030d0bfe kind: Event lastTimestamp: "2026-07-07T14:26:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:01Z" name: success-200-isvc-ab362-predictor.18c0081a35bcdb83 namespace: kserve-ci-e2e-test resourceVersion: "17655" uid: 0e326c77-41b3-49fb-b6c6-ed92908092c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: 0bebdfd1-90c8-4a87-b6f6-82ea0b88d508 kind: Event lastTimestamp: "2026-07-07T14:26:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:26:20Z" name: success-200-isvc-ab362.18c0081337f154ae namespace: kserve-ci-e2e-test resourceVersion: "17812" uid: cc498898-1bd8-4eac-8fd4-147895fa317a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: 0bebdfd1-90c8-4a87-b6f6-82ea0b88d508 kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-ab362": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ab362": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:31Z" name: success-200-isvc-ab362.18c0081338f01161 namespace: kserve-ci-e2e-test resourceVersion: "17255" uid: 25416df0-51a1-4e16-a7d5-9bae3f5d52a8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: 0bebdfd1-90c8-4a87-b6f6-82ea0b88d508 kind: Event lastTimestamp: "2026-07-07T14:25:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ab362": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:31Z" name: success-200-isvc-ab362.18c0081338f0589f namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 3f23ad5e-2144-4c16-8236-1244f4ba0e53 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab362 namespace: kserve-ci-e2e-test resourceVersion: "17809" uid: 0bebdfd1-90c8-4a87-b6f6-82ea0b88d508 kind: Event lastTimestamp: "2026-07-07T14:26:20Z" message: InferenceService [success-200-isvc-ab362] is Ready metadata: creationTimestamp: "2026-07-07T14:26:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:26:20Z" name: success-200-isvc-ab362.18c0081e9e4bd64e namespace: kserve-ci-e2e-test resourceVersion: "17811" uid: 878634b8-7ec2-4709-a119-fe47ded7c03a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:35:56.018364Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22525" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:35:56Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a4b97d4d5c namespace: kserve-ci-e2e-test resourceVersion: "22532" uid: 1b1bbf92-13d1-4017-8da4-f18fb3e4a24e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f31ce-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:56Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a4c4fe9923 namespace: kserve-ci-e2e-test resourceVersion: "22572" uid: dce8515d-2c2f-4812-8d26-0c863b81f766 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:35:57Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:35:57Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a4f7424c1e namespace: kserve-ci-e2e-test resourceVersion: "22595" uid: ebfe331e-59f6-4fa2-ac73-491ce3fedd03 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:35:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:57Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a4f8c4b3da namespace: kserve-ci-e2e-test resourceVersion: "22600" uid: d0e01f7b-33b3-41e2-8311-5ff88a1ea831 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:35:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:57Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a4fbb567c9 namespace: kserve-ci-e2e-test resourceVersion: "22602" uid: 00430d91-32f1-427d-ba17-42c727abddd0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:35:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:57Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a4fc2ec736 namespace: kserve-ci-e2e-test resourceVersion: "22603" uid: 73af7f75-22e2-428f-8c0a-94f60eb1b700 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:35:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:57Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a4fc3950d0 namespace: kserve-ci-e2e-test resourceVersion: "22604" uid: 3283b4e2-4fab-4b5d-985d-212022b8aecd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:35:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:57Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a4ff5bc46c namespace: kserve-ci-e2e-test resourceVersion: "22606" uid: ce0feb8d-e411-4f6a-9a91-037f0cc608e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:35:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:57Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a4ffef62e4 namespace: kserve-ci-e2e-test resourceVersion: "22607" uid: ca261c5c-9d07-43c0-8622-034455b8a06d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T14:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:45:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:13Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c008a50833ceec namespace: kserve-ci-e2e-test resourceVersion: "27190" uid: 12cc45a2-4d70-4748-af2d-d51bb83a7987 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:45:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:10Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c00925ee050f50 namespace: kserve-ci-e2e-test resourceVersion: "27074" uid: 9d752f99-a17a-4bbe-bf81-f62130eb7374 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:45:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:10Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c00925ee05c652 namespace: kserve-ci-e2e-test resourceVersion: "27076" uid: 4a688549-ce89-4113-ade0-1facb2e0c6a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:45:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: f6f46bc9-feae-4385-8cda-0f73e34675fa kind: Event lastTimestamp: "2026-07-07T14:45:13Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:45:13Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb.18c009267c8c3132 namespace: kserve-ci-e2e-test resourceVersion: "27189" uid: a785f6f2-cb3b-4205-999b-5e88baf0a557 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f31ce-predictor-5f44c97f5c namespace: kserve-ci-e2e-test resourceVersion: "22509" uid: 5656828f-1e1a-4bc6-8313-bff55858dc54 kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: 'Created pod: success-200-isvc-f31ce-predictor-5f44c97f5c-k6ccb' metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:56Z" name: success-200-isvc-f31ce-predictor-5f44c97f5c.18c008a4b910130d namespace: kserve-ci-e2e-test resourceVersion: "22527" uid: 760c1bb6-6d5d-42df-8248-cfebdb66f23e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22504" uid: af994a49-9e25-4d70-8c41-5b002e9c8765 kind: Event lastTimestamp: "2026-07-07T14:35:55Z" message: Scaled up replica set success-200-isvc-f31ce-predictor-5f44c97f5c from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:55Z" name: success-200-isvc-f31ce-predictor.18c008a4b73a8d63 namespace: kserve-ci-e2e-test resourceVersion: "22515" uid: e2573f0d-a844-4104-bb5d-0ece603a5131 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:36:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: a49d0d93-4e9c-482b-bd59-46481e8f1506 kind: Event lastTimestamp: "2026-07-07T14:36:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:41Z" name: success-200-isvc-f31ce-predictor.18c008a8364dd06c namespace: kserve-ci-e2e-test resourceVersion: "23005" uid: a8377522-6ccc-4038-9883-077be29cd09f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: a49d0d93-4e9c-482b-bd59-46481e8f1506 kind: Event lastTimestamp: "2026-07-07T14:36:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:41Z" name: success-200-isvc-f31ce-predictor.18c008a836b817ab namespace: kserve-ci-e2e-test resourceVersion: "23006" uid: 07b1eda0-5d13-41a3-a4ad-c7d313cd5795 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:36:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22768" uid: a49d0d93-4e9c-482b-bd59-46481e8f1506 kind: Event lastTimestamp: "2026-07-07T14:37:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:37:11Z" name: success-200-isvc-f31ce-predictor.18c008b2b3af8637 namespace: kserve-ci-e2e-test resourceVersion: "23275" uid: 69d8243c-aad3-4aa6-ac3b-915a97e2e60e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:36:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f31ce-predictor namespace: kserve-ci-e2e-test resourceVersion: "22768" uid: a49d0d93-4e9c-482b-bd59-46481e8f1506 kind: Event lastTimestamp: "2026-07-07T14:37:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:37:11Z" name: success-200-isvc-f31ce-predictor.18c008b2b432c1ff namespace: kserve-ci-e2e-test resourceVersion: "23276" uid: 85ec76fb-aba0-4d6d-ba46-83576448ddba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:35:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f31ce namespace: kserve-ci-e2e-test resourceVersion: "22484" uid: 47c93711-040c-4f0c-816d-470129726b42 kind: Event lastTimestamp: "2026-07-07T14:36:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:36:43Z" name: success-200-isvc-f31ce.18c008a4b73e6bf7 namespace: kserve-ci-e2e-test resourceVersion: "23041" uid: e6450ece-8395-437a-8849-40384dd8d782 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:36:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f31ce namespace: kserve-ci-e2e-test resourceVersion: "23038" uid: 47c93711-040c-4f0c-816d-470129726b42 kind: Event lastTimestamp: "2026-07-07T14:36:43Z" message: InferenceService [success-200-isvc-f31ce] is Ready metadata: creationTimestamp: "2026-07-07T14:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:36:43Z" name: success-200-isvc-f31ce.18c008afc4857688 namespace: kserve-ci-e2e-test resourceVersion: "23040" uid: 69177bed-2f48-44bf-af5a-35c4e222d124 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:34:45.754377Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:34:45Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c008945d6e00d4 namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 8c701a01-b46b-454e-87eb-ab002c383491 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:34:46Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c00894773afeaa namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: da9b0a04-4546-4e99-af5a-3e45af6adb34 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c0089478e1576c namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: f331ca12-b7d2-49e0-9d9a-3503091c641d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c008947b9e75c1 namespace: kserve-ci-e2e-test resourceVersion: "21618" uid: 3a333df5-a78b-4556-87e9-bcf6881fff81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c008947c302f58 namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: ec0b72a5-aeca-4a53-a496-d6a19d0aeacf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c008947c3bc2b3 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: 49513ced-badc-4dc5-8102-98532bd53216 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c008947f44dbd6 namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 6083abc5-b5a9-4ca1-9d39-313b3c21f56d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:34:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:46Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c008947fbee1d8 namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: a0d48172-479e-400c-a1b4-f99a54191f78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:34:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:35:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:34:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:24Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c00894e5fce30c namespace: kserve-ci-e2e-test resourceVersion: "22146" uid: 2f07a807-80c5-4c82-ab8b-fc684062e423 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:56Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c008a4b97f8071 namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: 4d8c232e-e2a4-4df7-8893-62545039b4a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:35:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:56Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c008a4b980b24e namespace: kserve-ci-e2e-test resourceVersion: "22537" uid: a6a0eb4e-b948-48c6-abf7-81a5903f5699 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: fd4bd77b-9822-4829-8e54-d6e24b6f264c kind: Event lastTimestamp: "2026-07-07T14:35:59Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:59Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv.18c008a56e193011 namespace: kserve-ci-e2e-test resourceVersion: "22630" uid: 6330fb0a-7343-4851-bfc4-3a772c29a3df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fbc26-predictor-b5c7dd9f9 namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 26b6a558-b4a7-4849-ba0e-73a604f35653 kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: 'Created pod: success-200-isvc-fbc26-predictor-b5c7dd9f9-gkdxv' metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:45Z" name: success-200-isvc-fbc26-predictor-b5c7dd9f9.18c008945cd84966 namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: da59cd9c-4b2d-4bc6-8246-aeee6f7b1586 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: b3aa9073-4df1-4dbe-b3af-8068d84c2bf6 kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: Scaled up replica set success-200-isvc-fbc26-predictor-b5c7dd9f9 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:34:45Z" name: success-200-isvc-fbc26-predictor.18c008945adac389 namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: adcd6307-36e5-486e-8e69-02f8db0f745f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 4bb64087-ee08-4f53-a2ad-ed4a568a429b kind: Event lastTimestamp: "2026-07-07T14:35:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:15Z" name: success-200-isvc-fbc26-predictor.18c00897d960ca45 namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: 97471343-d6d0-4b30-bafb-d4b3bd9d49f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 4bb64087-ee08-4f53-a2ad-ed4a568a429b kind: Event lastTimestamp: "2026-07-07T14:35:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:15Z" name: success-200-isvc-fbc26-predictor.18c00897d9c9fb13 namespace: kserve-ci-e2e-test resourceVersion: "22075" uid: c05c8ea1-1746-4c0d-ab0f-92d949609f8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: 4bb64087-ee08-4f53-a2ad-ed4a568a429b kind: Event lastTimestamp: "2026-07-07T14:35:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:35:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:45Z" name: success-200-isvc-fbc26-predictor.18c0089ed85f30b0 namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: 6322950b-2873-4239-b6fc-5c484df7f107 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:35:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbc26-predictor namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: 4bb64087-ee08-4f53-a2ad-ed4a568a429b kind: Event lastTimestamp: "2026-07-07T14:35:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:35:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:45Z" name: success-200-isvc-fbc26-predictor.18c0089ed8e24c98 namespace: kserve-ci-e2e-test resourceVersion: "22354" uid: 22d0efd9-fc44-404a-9d7a-1aba3e70e10f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 221f1354-1827-4388-86d9-c0d0b5d91b42 kind: Event lastTimestamp: "2026-07-07T14:35:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:34Z" name: success-200-isvc-fbc26.18c008945ae9e55c namespace: kserve-ci-e2e-test resourceVersion: "22240" uid: 0a276404-2111-4c13-af8e-9c8ecb82dbd9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 221f1354-1827-4388-86d9-c0d0b5d91b42 kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-fbc26": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fbc26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:34:45Z" name: success-200-isvc-fbc26.18c008945bb314c6 namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: 37cfb859-93e0-45a6-896b-c5d079f7f18d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 221f1354-1827-4388-86d9-c0d0b5d91b42 kind: Event lastTimestamp: "2026-07-07T14:34:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fbc26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:34:45Z" name: success-200-isvc-fbc26.18c008945bb361e3 namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 945d8c7a-cf3b-46e0-bca8-f40a439c03c8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbc26 namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 221f1354-1827-4388-86d9-c0d0b5d91b42 kind: Event lastTimestamp: "2026-07-07T14:35:34Z" message: InferenceService [success-200-isvc-fbc26] is Ready metadata: creationTimestamp: "2026-07-07T14:35:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:34Z" name: success-200-isvc-fbc26.18c0089fa16d391b namespace: kserve-ci-e2e-test resourceVersion: "22239" uid: 9552b19a-7ba1-4414-b470-12bc98012705 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:35:06.307276Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21894" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:35:06Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c00899267a4b53 namespace: kserve-ci-e2e-test resourceVersion: "21901" uid: c316db94-5df3-43c4-9aa4-a39a4644b12f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-fe15b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:06Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c008993271a708 namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: 7779d24c-be83-4ed2-a8f0-a0c7e83f31e4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21912" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:35:07Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c008996451f270 namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: 8614cd84-e509-4e3c-b95f-4c1411044170 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c0089966100b5a namespace: kserve-ci-e2e-test resourceVersion: "21974" uid: 06bfd181-9268-4399-8fc2-f1e007bf9ffd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c00899692b92f6 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 15d16122-88fd-49b2-899f-98c1ca4eb292 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c00899699e7b24 namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: e0c974e6-3143-4d9b-811c-e20e6129d305 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c0089969a929dc namespace: kserve-ci-e2e-test resourceVersion: "21978" uid: dc1f50c6-263b-42e7-bbfc-0e1d70722164 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c008996cba14e3 namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: 0b6cf75a-f318-41ab-9f64-0e0986f951d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:35:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:07Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c008996d352b56 namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 8942240b-eeda-47f5-9168-1b103f28aacf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:35:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:35:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:35:45Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c00899cefe61e3 namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: f598bd15-f638-4bb8-b011-edf4b3e5ba51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c0091a5e6b40c4 namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: 201a8786-779a-4969-8b4a-650dcfe6e2ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:44:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 963cf95d-8dda-4225-b8e7-d945024f7253 kind: Event lastTimestamp: "2026-07-07T14:44:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:44:21Z" name: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4.18c0091a5e6c18d6 namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: e75bfbc9-d2a5-4298-af25-77480d194372 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fe15b-predictor-6654f94d55 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: 630a2df6-455f-4cd4-a4f4-f3be571acb2f kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: 'Created pod: success-200-isvc-fe15b-predictor-6654f94d55-dmfx4' metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:06Z" name: success-200-isvc-fe15b-predictor-6654f94d55.18c00899260f2088 namespace: kserve-ci-e2e-test resourceVersion: "21896" uid: 06be01e9-ca4c-4c00-afe4-abf60e5ee3f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: 8fb036c9-86d1-42d9-93d6-1165180bf069 kind: Event lastTimestamp: "2026-07-07T14:35:06Z" message: Scaled up replica set success-200-isvc-fe15b-predictor-6654f94d55 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:06Z" name: success-200-isvc-fe15b-predictor.18c00899244650c7 namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: 302b57bf-f4b7-4993-b898-d7626a821c34 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: a6512470-8a3d-4b20-b331-d2dde1c97c69 kind: Event lastTimestamp: "2026-07-07T14:35:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:21Z" name: success-200-isvc-fe15b-predictor.18c0089ca2d6521f namespace: kserve-ci-e2e-test resourceVersion: "22121" uid: f7326860-cb56-4eee-8cd7-7a9981d9a35e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: a6512470-8a3d-4b20-b331-d2dde1c97c69 kind: Event lastTimestamp: "2026-07-07T14:35:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:35:21Z" name: success-200-isvc-fe15b-predictor.18c0089ca3546c00 namespace: kserve-ci-e2e-test resourceVersion: "22123" uid: 70ea0e7e-8739-4268-93cd-832c9da5896c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:35:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22122" uid: a6512470-8a3d-4b20-b331-d2dde1c97c69 kind: Event lastTimestamp: "2026-07-07T14:36:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:21Z" name: success-200-isvc-fe15b-predictor.18c008a0220eabea namespace: kserve-ci-e2e-test resourceVersion: "22850" uid: e78d40e6-6a44-4fff-9b86-1cbd95df8ae6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:35:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe15b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22122" uid: a6512470-8a3d-4b20-b331-d2dde1c97c69 kind: Event lastTimestamp: "2026-07-07T14:36:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:36:21Z" name: success-200-isvc-fe15b-predictor.18c008a0226f4fae namespace: kserve-ci-e2e-test resourceVersion: "22851" uid: 630cdc7a-6346-4d5a-88ee-483e87baf4e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:35:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fe15b namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: 004f49b8-f2ec-47ad-a127-41a1c96d6023 kind: Event lastTimestamp: "2026-07-07T14:35:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:55Z" name: success-200-isvc-fe15b.18c0089924496b95 namespace: kserve-ci-e2e-test resourceVersion: "22462" uid: c1f9b5f6-d3ff-40cc-b95c-515760a1675b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:35:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fe15b namespace: kserve-ci-e2e-test resourceVersion: "22460" uid: 004f49b8-f2ec-47ad-a127-41a1c96d6023 kind: Event lastTimestamp: "2026-07-07T14:35:55Z" message: InferenceService [success-200-isvc-fe15b] is Ready metadata: creationTimestamp: "2026-07-07T14:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:35:55Z" name: success-200-isvc-fe15b.18c008a48a276869 namespace: kserve-ci-e2e-test resourceVersion: "22461" uid: 7dbd065c-e411-4663-8b4d-624ca5d4a62d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:46:11.032724Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-36f00-7bc7bf4797-g5dbt namespace: kserve-ci-e2e-test resourceVersion: "27882" uid: 20d3e8bd-7666-48aa-9b00-a9523ace4547 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-36f00-7bc7bf4797-g5dbt to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:46:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:46:11Z" name: switch-graph-36f00-7bc7bf4797-g5dbt.18c00933eb33b800 namespace: kserve-ci-e2e-test resourceVersion: "27887" uid: e87268c2-a5c3-486e-9663-aea49982fa60 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-36f00-7bc7bf4797-g5dbt namespace: kserve-ci-e2e-test resourceVersion: "27885" uid: 20d3e8bd-7666-48aa-9b00-a9523ace4547 kind: Event lastTimestamp: "2026-07-07T14:46:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-36f00-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:11Z" name: switch-graph-36f00-7bc7bf4797-g5dbt.18c00933f9419ce0 namespace: kserve-ci-e2e-test resourceVersion: "27904" uid: 962b4e5c-152b-4ad1-8ce0-91062f7c4bdd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:12Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-36f00-7bc7bf4797-g5dbt namespace: kserve-ci-e2e-test resourceVersion: "27891" uid: 20d3e8bd-7666-48aa-9b00-a9523ace4547 kind: Event lastTimestamp: "2026-07-07T14:46:12Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:46:12Z" name: switch-graph-36f00-7bc7bf4797-g5dbt.18c0093428b01494 namespace: kserve-ci-e2e-test resourceVersion: "27910" uid: 1f2a6336-a30c-4ac7-a29a-10773a1c8305 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-36f00} kind: Pod name: switch-graph-36f00-7bc7bf4797-g5dbt namespace: kserve-ci-e2e-test resourceVersion: "27885" uid: 20d3e8bd-7666-48aa-9b00-a9523ace4547 kind: Event lastTimestamp: "2026-07-07T14:46:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:12Z" name: switch-graph-36f00-7bc7bf4797-g5dbt.18c0093429febb47 namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 89439fe2-ed12-4602-8dcc-2318ded1e59d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-36f00} kind: Pod name: switch-graph-36f00-7bc7bf4797-g5dbt namespace: kserve-ci-e2e-test resourceVersion: "27885" uid: 20d3e8bd-7666-48aa-9b00-a9523ace4547 kind: Event lastTimestamp: "2026-07-07T14:46:12Z" message: 'Created container: switch-graph-36f00' metadata: creationTimestamp: "2026-07-07T14:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:12Z" name: switch-graph-36f00-7bc7bf4797-g5dbt.18c009342d2ce4e0 namespace: kserve-ci-e2e-test resourceVersion: "27914" uid: 656d5526-4851-44ad-bdf2-3598baa4acc0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-36f00} kind: Pod name: switch-graph-36f00-7bc7bf4797-g5dbt namespace: kserve-ci-e2e-test resourceVersion: "27885" uid: 20d3e8bd-7666-48aa-9b00-a9523ace4547 kind: Event lastTimestamp: "2026-07-07T14:46:12Z" message: Started container switch-graph-36f00 metadata: creationTimestamp: "2026-07-07T14:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:46:12Z" name: switch-graph-36f00-7bc7bf4797-g5dbt.18c009342dbf5a43 namespace: kserve-ci-e2e-test resourceVersion: "27915" uid: 253523ed-f1ac-497a-92b8-7fdbb511a611 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:02:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-36f00} kind: Pod name: switch-graph-36f00-7bc7bf4797-g5dbt namespace: kserve-ci-e2e-test resourceVersion: "27885" uid: 20d3e8bd-7666-48aa-9b00-a9523ace4547 kind: Event lastTimestamp: "2026-07-07T15:02:30Z" message: Stopping container switch-graph-36f00 metadata: creationTimestamp: "2026-07-07T15:02:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:02:30Z" name: switch-graph-36f00-7bc7bf4797-g5dbt.18c00a17e2409cc7 namespace: kserve-ci-e2e-test resourceVersion: "34760" uid: 6fd5b47c-5ae2-4933-a745-952a350e6d86 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:02:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-36f00} kind: Pod name: switch-graph-36f00-7bc7bf4797-g5dbt namespace: kserve-ci-e2e-test resourceVersion: "27885" uid: 20d3e8bd-7666-48aa-9b00-a9523ace4547 kind: Event lastTimestamp: "2026-07-07T15:02:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:02:58Z" name: switch-graph-36f00-7bc7bf4797-g5dbt.18c00a18b7408b3a namespace: kserve-ci-e2e-test resourceVersion: "35039" uid: 557d5086-6db3-40f4-bb74-ad9933121e3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-36f00-7bc7bf4797 namespace: kserve-ci-e2e-test resourceVersion: "27879" uid: 25e91603-84a7-4f8f-b5de-11a53a5852d7 kind: Event lastTimestamp: "2026-07-07T14:46:11Z" message: 'Created pod: switch-graph-36f00-7bc7bf4797-g5dbt' metadata: creationTimestamp: "2026-07-07T14:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:11Z" name: switch-graph-36f00-7bc7bf4797.18c00933eace3dbc namespace: kserve-ci-e2e-test resourceVersion: "27884" uid: 9bab2b24-09b2-4955-a7de-7b41cf9da7cb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27877" uid: e94de03a-0b69-445a-9d47-4617697372e0 kind: Event lastTimestamp: "2026-07-07T14:46:11Z" message: Scaled up replica set switch-graph-36f00-7bc7bf4797 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:11Z" name: switch-graph-36f00.18c00933ea05fdff namespace: kserve-ci-e2e-test resourceVersion: "27880" uid: d6d70311-c709-4638-a883-ca0283193dd2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27897" uid: 1304b222-8cb6-4e26-a26c-9df61ab2de08 kind: Event lastTimestamp: "2026-07-07T14:46:11Z" message: InferenceGraph [switch-graph-36f00] is Ready metadata: creationTimestamp: "2026-07-07T14:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:46:11Z" name: switch-graph-36f00.18c00933ef453892 namespace: kserve-ci-e2e-test resourceVersion: "27899" uid: efd08207-d728-44c5-b3dc-920f3f0891bf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27869" uid: 1304b222-8cb6-4e26-a26c-9df61ab2de08 kind: Event lastTimestamp: "2026-07-07T14:46:11Z" message: 'Failed to update status for InferenceGraph "switch-graph-36f00": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-36f00": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:46:11Z" name: switch-graph-36f00.18c00933f190e457 namespace: kserve-ci-e2e-test resourceVersion: "27900" uid: 191a62ec-a6b6-4be2-b92f-461180feaf73 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27869" uid: 1304b222-8cb6-4e26-a26c-9df61ab2de08 kind: Event lastTimestamp: "2026-07-07T14:46:11Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-36f00": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:46:11Z" name: switch-graph-36f00.18c00933f19133d9 namespace: kserve-ci-e2e-test resourceVersion: "27901" uid: 842603bf-7b92-41bb-8d1d-6112cf17d70a reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27878" uid: 6a58a16a-71cf-4a68-9d04-aa502c80ed17 kind: Event lastTimestamp: "2026-07-07T14:46:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:26Z" name: switch-graph-36f00.18c00937686455de namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: ad02e59c-d281-4a68-aec3-1ad89239264e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:46:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-36f00 namespace: kserve-ci-e2e-test resourceVersion: "27878" uid: 6a58a16a-71cf-4a68-9d04-aa502c80ed17 kind: Event lastTimestamp: "2026-07-07T14:46:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:26Z" name: switch-graph-36f00.18c0093768c8e805 namespace: kserve-ci-e2e-test resourceVersion: "28052" uid: cff4f0e7-cdc3-443a-9e7d-912d8bef29de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-36f00 namespace: kserve-ci-e2e-test resourceVersion: "28051" uid: 6a58a16a-71cf-4a68-9d04-aa502c80ed17 kind: Event lastTimestamp: "2026-07-07T14:46:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:56Z" name: switch-graph-36f00.18c0093ae7739e81 namespace: kserve-ci-e2e-test resourceVersion: "28312" uid: 5cc6b6f8-c77a-4888-9b50-22817857b6ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-36f00 namespace: kserve-ci-e2e-test resourceVersion: "28051" uid: 6a58a16a-71cf-4a68-9d04-aa502c80ed17 kind: Event lastTimestamp: "2026-07-07T14:46:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:46:56Z" name: switch-graph-36f00.18c0093ae7d5e875 namespace: kserve-ci-e2e-test resourceVersion: "28313" uid: 2abc4a48-f53d-4b27-830c-26ed45b37fb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:25:41.100949Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-520df-7fcf5f5875-w848t namespace: kserve-ci-e2e-test resourceVersion: "17465" uid: c77788d1-a4e8-4fda-aba3-5f7cc5558318 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-520df-7fcf5f5875-w848t to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:25:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:25:41Z" name: switch-graph-520df-7fcf5f5875-w848t.18c008158d8e22c1 namespace: kserve-ci-e2e-test resourceVersion: "17471" uid: a1fccf87-9fc9-43e2-b188-bd93ff59f35d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:41Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-520df-7fcf5f5875-w848t namespace: kserve-ci-e2e-test resourceVersion: "17475" uid: c77788d1-a4e8-4fda-aba3-5f7cc5558318 kind: Event lastTimestamp: "2026-07-07T14:25:41Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:25:41Z" name: switch-graph-520df-7fcf5f5875-w848t.18c00815a778623a namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: f1a95fd9-05c0-4f58-bd19-e5518ebfbdff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-520df} kind: Pod name: switch-graph-520df-7fcf5f5875-w848t namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: c77788d1-a4e8-4fda-aba3-5f7cc5558318 kind: Event lastTimestamp: "2026-07-07T14:25:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:41Z" name: switch-graph-520df-7fcf5f5875-w848t.18c00815a91462aa namespace: kserve-ci-e2e-test resourceVersion: "17487" uid: 23fcc2d3-2744-4606-ab2a-43aa0f307e91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-520df} kind: Pod name: switch-graph-520df-7fcf5f5875-w848t namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: c77788d1-a4e8-4fda-aba3-5f7cc5558318 kind: Event lastTimestamp: "2026-07-07T14:25:41Z" message: 'Created container: switch-graph-520df' metadata: creationTimestamp: "2026-07-07T14:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:41Z" name: switch-graph-520df-7fcf5f5875-w848t.18c00815ac99a81e namespace: kserve-ci-e2e-test resourceVersion: "17489" uid: b079bbb2-cd07-4bb5-9fef-849d5ddca473 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-520df} kind: Pod name: switch-graph-520df-7fcf5f5875-w848t namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: c77788d1-a4e8-4fda-aba3-5f7cc5558318 kind: Event lastTimestamp: "2026-07-07T14:25:41Z" message: Started container switch-graph-520df metadata: creationTimestamp: "2026-07-07T14:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:25:41Z" name: switch-graph-520df-7fcf5f5875-w848t.18c00815ad10afce namespace: kserve-ci-e2e-test resourceVersion: "17490" uid: 893e9232-f806-492a-9302-fd042913e839 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-520df} kind: Pod name: switch-graph-520df-7fcf5f5875-w848t namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: c77788d1-a4e8-4fda-aba3-5f7cc5558318 kind: Event lastTimestamp: "2026-07-07T14:33:55Z" message: Stopping container switch-graph-520df metadata: creationTimestamp: "2026-07-07T14:33:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:33:55Z" name: switch-graph-520df-7fcf5f5875-w848t.18c00888bf62a3e5 namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 5c4bed99-8c2e-41bf-b10e-52528b3b1d2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-520df} kind: Pod name: switch-graph-520df-7fcf5f5875-w848t namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: c77788d1-a4e8-4fda-aba3-5f7cc5558318 kind: Event lastTimestamp: "2026-07-07T14:34:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:22Z" name: switch-graph-520df-7fcf5f5875-w848t.18c008893c3c06a1 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: d85f1daa-0ae5-463a-b594-29a4b6726dea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-520df-7fcf5f5875 namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: 26c91217-0eef-4658-b35a-9815e6a26500 kind: Event lastTimestamp: "2026-07-07T14:25:41Z" message: 'Created pod: switch-graph-520df-7fcf5f5875-w848t' metadata: creationTimestamp: "2026-07-07T14:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:41Z" name: switch-graph-520df-7fcf5f5875.18c008158ba2bd9a namespace: kserve-ci-e2e-test resourceVersion: "17468" uid: 06080e54-d6c9-4b3f-b7dd-b679880bbced reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-520df namespace: kserve-ci-e2e-test resourceVersion: "17460" uid: a5d0885e-9f91-4601-879b-307afaebf495 kind: Event lastTimestamp: "2026-07-07T14:25:41Z" message: Scaled up replica set switch-graph-520df-7fcf5f5875 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:41Z" name: switch-graph-520df.18c008158aa49963 namespace: kserve-ci-e2e-test resourceVersion: "17463" uid: 21374dd0-d610-4f7a-afc6-68c82dadb645 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-520df namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: ec77936a-e35f-4488-93ae-15ac9dc0fd48 kind: Event lastTimestamp: "2026-07-07T14:25:41Z" message: InferenceGraph [switch-graph-520df] is Ready metadata: creationTimestamp: "2026-07-07T14:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:41Z" name: switch-graph-520df.18c0081594547e05 namespace: kserve-ci-e2e-test resourceVersion: "17481" uid: 46e52546-d6f5-4888-9f04-944b4331426b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:25:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-520df namespace: kserve-ci-e2e-test resourceVersion: "17461" uid: db3066c5-3e9b-40f8-adc8-3631dda1cd93 kind: Event lastTimestamp: "2026-07-07T14:26:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:26Z" name: switch-graph-520df.18c008190908e437 namespace: kserve-ci-e2e-test resourceVersion: "17852" uid: 8d079a78-a8aa-43b0-bb3b-281ad9050c8f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:25:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-520df namespace: kserve-ci-e2e-test resourceVersion: "17461" uid: db3066c5-3e9b-40f8-adc8-3631dda1cd93 kind: Event lastTimestamp: "2026-07-07T14:26:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:26Z" name: switch-graph-520df.18c00819098c01e1 namespace: kserve-ci-e2e-test resourceVersion: "17853" uid: fa7c01d8-39b9-424a-98c0-f63762975cc7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:24:20.640307Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a2dea-694795ff54-nc86f namespace: kserve-ci-e2e-test resourceVersion: "16396" uid: 66870c1d-06f9-42b0-976f-eae6b1256da3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-a2dea-694795ff54-nc86f to ip-10-0-138-82.ec2.internal metadata: creationTimestamp: "2026-07-07T14:24:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:24:20Z" name: switch-graph-a2dea-694795ff54-nc86f.18c00802d1ba4a5d namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: e608e69e-3d1b-4cc0-a58f-96eb7ca8d09d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58d9bbc898-wwkq8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a2dea-694795ff54-nc86f namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: 66870c1d-06f9-42b0-976f-eae6b1256da3 kind: Event lastTimestamp: "2026-07-07T14:24:21Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:24:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:24:21Z" name: switch-graph-a2dea-694795ff54-nc86f.18c00802eb3be226 namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: cb935967-5a77-4eb3-9162-72e59f078aba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2dea} kind: Pod name: switch-graph-a2dea-694795ff54-nc86f namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 66870c1d-06f9-42b0-976f-eae6b1256da3 kind: Event lastTimestamp: "2026-07-07T14:24:21Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" metadata: creationTimestamp: "2026-07-07T14:24:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:21Z" name: switch-graph-a2dea-694795ff54-nc86f.18c00802ec94bb2e namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 575a9b9a-8e24-472b-ad58-ef798c1df5b4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2dea} kind: Pod name: switch-graph-a2dea-694795ff54-nc86f namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 66870c1d-06f9-42b0-976f-eae6b1256da3 kind: Event lastTimestamp: "2026-07-07T14:24:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" in 1.93s (1.93s including waiting). Image size: 218072702 bytes.' metadata: creationTimestamp: "2026-07-07T14:24:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:23Z" name: switch-graph-a2dea-694795ff54-nc86f.18c008035fa472e1 namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 5dba6a2e-9a4d-48e0-916e-eef79207d23b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2dea} kind: Pod name: switch-graph-a2dea-694795ff54-nc86f namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 66870c1d-06f9-42b0-976f-eae6b1256da3 kind: Event lastTimestamp: "2026-07-07T14:24:23Z" message: 'Created container: switch-graph-a2dea' metadata: creationTimestamp: "2026-07-07T14:24:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:23Z" name: switch-graph-a2dea-694795ff54-nc86f.18c0080363b2c5c5 namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: e6b3d411-2ad2-4916-be03-7539a29cf47b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2dea} kind: Pod name: switch-graph-a2dea-694795ff54-nc86f namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 66870c1d-06f9-42b0-976f-eae6b1256da3 kind: Event lastTimestamp: "2026-07-07T14:24:23Z" message: Started container switch-graph-a2dea metadata: creationTimestamp: "2026-07-07T14:24:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:23Z" name: switch-graph-a2dea-694795ff54-nc86f.18c00803642fa9a7 namespace: kserve-ci-e2e-test resourceVersion: "16430" uid: 2cf3a0f4-9de0-4a23-93ae-fdef3301870d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2dea} kind: Pod name: switch-graph-a2dea-694795ff54-nc86f namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 66870c1d-06f9-42b0-976f-eae6b1256da3 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Stopping container switch-graph-a2dea metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: switch-graph-a2dea-694795ff54-nc86f.18c0080530e6d007 namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: 54ee9fbd-fb45-4039-8fbf-ded7b2a1d33e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2dea} kind: Pod name: switch-graph-a2dea-694795ff54-nc86f namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 66870c1d-06f9-42b0-976f-eae6b1256da3 kind: Event lastTimestamp: "2026-07-07T14:24:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:59Z" name: switch-graph-a2dea-694795ff54-nc86f.18c0080614ac51c0 namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: ddfad1be-322b-4f0c-8748-63d1ad3814b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-82.ec2.internal source: component: kubelet host: ip-10-0-138-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-a2dea-694795ff54 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: dc9b8459-499e-40ef-b37b-46d94a327835 kind: Event lastTimestamp: "2026-07-07T14:24:20Z" message: 'Created pod: switch-graph-a2dea-694795ff54-nc86f' metadata: creationTimestamp: "2026-07-07T14:24:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:20Z" name: switch-graph-a2dea-694795ff54.18c00802d1373f53 namespace: kserve-ci-e2e-test resourceVersion: "16398" uid: 520a0d69-3610-4d06-a76d-752ad967041b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-a2dea namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: cabb01f0-bea7-443a-b7ff-5aa564d6af0e kind: Event lastTimestamp: "2026-07-07T14:24:20Z" message: Scaled up replica set switch-graph-a2dea-694795ff54 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:24:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:20Z" name: switch-graph-a2dea.18c00802cffe3272 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 218bd09c-f72c-46fa-99e4-58879ea60b22 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-a2dea namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: 6366eaa8-30d1-42a1-9e28-2dee4c4f9ec7 kind: Event lastTimestamp: "2026-07-07T14:24:20Z" message: InferenceGraph [switch-graph-a2dea] is Ready metadata: creationTimestamp: "2026-07-07T14:24:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:20Z" name: switch-graph-a2dea.18c00802d5f518ad namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 7bb78654-0415-4654-8c88-3f71a6d2bb3c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "35055"