--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:26:31.161200Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-007a5-866fd95dbf-w2kn6 namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: 13f28c0b-3a9c-4fc3-b1f9-67f11f2be0a0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-007a5-866fd95dbf-w2kn6 to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:26:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:26:31Z" name: ensemble-graph-007a5-866fd95dbf-w2kn6.18b32a4dbfdc4006 namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: f77a7139-c9ac-4330-b45f-00a3a70c3416 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:31Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-007a5-866fd95dbf-w2kn6 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 13f28c0b-3a9c-4fc3-b1f9-67f11f2be0a0 kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-007a5-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:31Z" name: ensemble-graph-007a5-866fd95dbf-w2kn6.18b32a4dcb8111db namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 9e3d0cfb-24cd-4d34-bd39-0702443e6615 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:32Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-007a5-866fd95dbf-w2kn6 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 13f28c0b-3a9c-4fc3-b1f9-67f11f2be0a0 kind: Event lastTimestamp: "2026-05-26T16:26:32Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:26:32Z" name: ensemble-graph-007a5-866fd95dbf-w2kn6.18b32a4dfd18bea0 namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 622e2879-e63e-47bb-aae8-ed8a5660f612 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-007a5} kind: Pod name: ensemble-graph-007a5-866fd95dbf-w2kn6 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 13f28c0b-3a9c-4fc3-b1f9-67f11f2be0a0 kind: Event lastTimestamp: "2026-05-26T16:26:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:32Z" name: ensemble-graph-007a5-866fd95dbf-w2kn6.18b32a4dfe50bbce namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 9ed67bda-460c-4235-9918-c38f3f7d104c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-007a5} kind: Pod name: ensemble-graph-007a5-866fd95dbf-w2kn6 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 13f28c0b-3a9c-4fc3-b1f9-67f11f2be0a0 kind: Event lastTimestamp: "2026-05-26T16:26:32Z" message: 'Created container: ensemble-graph-007a5' metadata: creationTimestamp: "2026-05-26T16:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:32Z" name: ensemble-graph-007a5-866fd95dbf-w2kn6.18b32a4e01206d97 namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: 7a2ec335-a5fc-4fd2-8b67-c4f2e2a9ef0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-007a5} kind: Pod name: ensemble-graph-007a5-866fd95dbf-w2kn6 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 13f28c0b-3a9c-4fc3-b1f9-67f11f2be0a0 kind: Event lastTimestamp: "2026-05-26T16:26:32Z" message: Started container ensemble-graph-007a5 metadata: creationTimestamp: "2026-05-26T16:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:32Z" name: ensemble-graph-007a5-866fd95dbf-w2kn6.18b32a4e01aa4193 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 80713040-04e0-4c32-a784-d969c6d31eba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-007a5} kind: Pod name: ensemble-graph-007a5-866fd95dbf-w2kn6 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 13f28c0b-3a9c-4fc3-b1f9-67f11f2be0a0 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: Stopping container ensemble-graph-007a5 metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:41Z" name: ensemble-graph-007a5-866fd95dbf-w2kn6.18b32a5018ce0ca3 namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: 83b14e91-09fb-4dc1-9bb2-3036e603a2eb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:26:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-007a5} kind: Pod name: ensemble-graph-007a5-866fd95dbf-w2kn6 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 13f28c0b-3a9c-4fc3-b1f9-67f11f2be0a0 kind: Event lastTimestamp: "2026-05-26T16:27:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:08Z" name: ensemble-graph-007a5-866fd95dbf-w2kn6.18b32a50a7cee8ab namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 6c56f470-71b6-479f-9f76-0bd94d9f4c8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-007a5-866fd95dbf namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 31eb311d-a32f-485f-974c-8844de5d6b2e kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: 'Created pod: ensemble-graph-007a5-866fd95dbf-w2kn6' metadata: creationTimestamp: "2026-05-26T16:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:31Z" name: ensemble-graph-007a5-866fd95dbf.18b32a4dbf6fe70c namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 4eadd1fc-6a38-4bf2-9e61-7360fd50af89 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-007a5 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 5d64c0bf-c0ad-472d-8473-7c711a51b079 kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: Scaled up replica set ensemble-graph-007a5-866fd95dbf from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:31Z" name: ensemble-graph-007a5.18b32a4dbec84e0a namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 62f9b677-1378-43ce-b3ba-9b15c608e591 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-007a5 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 5fd93cf3-1642-46dd-a08b-a7d70ee274c8 kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: InferenceGraph [ensemble-graph-007a5] is Ready metadata: creationTimestamp: "2026-05-26T16:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:31Z" name: ensemble-graph-007a5.18b32a4dc28df1e0 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 263de99b-9575-4856-95bd-ff13d0121fa5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-007a5 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 5fd93cf3-1642-46dd-a08b-a7d70ee274c8 kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-007a5": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-007a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:31Z" name: ensemble-graph-007a5.18b32a4dc530fdc7 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: eaa6e506-8ada-46a4-8ddb-c62a9b8c7fbb reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-007a5 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 5fd93cf3-1642-46dd-a08b-a7d70ee274c8 kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-007a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:31Z" name: ensemble-graph-007a5.18b32a4dc5316276 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: 30ba916d-2f79-4c38-bce6-6d33798b5ba9 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:27:41.467328Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2138e-94f4bbc69-5zxlj namespace: kserve-ci-e2e-test resourceVersion: "22171" uid: fe976bc8-2285-426e-9c70-ff4c7e869605 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-2138e-94f4bbc69-5zxlj to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:27:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:27:41Z" name: ensemble-graph-2138e-94f4bbc69-5zxlj.18b32a5e1e6ea1fe namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: 91ea9e12-6a91-447d-98f6-e598591c4693 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:41Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2138e-94f4bbc69-5zxlj namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: fe976bc8-2285-426e-9c70-ff4c7e869605 kind: Event lastTimestamp: "2026-05-26T16:27:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-2138e-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:41Z" name: ensemble-graph-2138e-94f4bbc69-5zxlj.18b32a5e285bf86d namespace: kserve-ci-e2e-test resourceVersion: "22192" uid: 5f818896-53ca-43c2-bf42-ec074e2daec9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:42Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-2138e-94f4bbc69-5zxlj namespace: kserve-ci-e2e-test resourceVersion: "22180" uid: fe976bc8-2285-426e-9c70-ff4c7e869605 kind: Event lastTimestamp: "2026-05-26T16:27:42Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:27:42Z" name: ensemble-graph-2138e-94f4bbc69-5zxlj.18b32a5e5bba351f namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4421d4d5-74db-496f-b7fd-e0a803ede68b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2138e} kind: Pod name: ensemble-graph-2138e-94f4bbc69-5zxlj namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: fe976bc8-2285-426e-9c70-ff4c7e869605 kind: Event lastTimestamp: "2026-05-26T16:27:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:42Z" name: ensemble-graph-2138e-94f4bbc69-5zxlj.18b32a5e5ced7206 namespace: kserve-ci-e2e-test resourceVersion: "22202" uid: c2c1cb13-0601-44c7-9c58-8d9cb8905fad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2138e} kind: Pod name: ensemble-graph-2138e-94f4bbc69-5zxlj namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: fe976bc8-2285-426e-9c70-ff4c7e869605 kind: Event lastTimestamp: "2026-05-26T16:27:42Z" message: 'Created container: ensemble-graph-2138e' metadata: creationTimestamp: "2026-05-26T16:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:42Z" name: ensemble-graph-2138e-94f4bbc69-5zxlj.18b32a5e5f97d559 namespace: kserve-ci-e2e-test resourceVersion: "22203" uid: b7cb98f0-c398-45be-914e-532236a761d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2138e} kind: Pod name: ensemble-graph-2138e-94f4bbc69-5zxlj namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: fe976bc8-2285-426e-9c70-ff4c7e869605 kind: Event lastTimestamp: "2026-05-26T16:27:42Z" message: Started container ensemble-graph-2138e metadata: creationTimestamp: "2026-05-26T16:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:42Z" name: ensemble-graph-2138e-94f4bbc69-5zxlj.18b32a5e6007044e namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: e0c43c5b-513e-46d8-ac93-96b347c7468b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2138e} kind: Pod name: ensemble-graph-2138e-94f4bbc69-5zxlj namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: fe976bc8-2285-426e-9c70-ff4c7e869605 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Stopping container ensemble-graph-2138e metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: ensemble-graph-2138e-94f4bbc69-5zxlj.18b32ad147ef2d6c namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: e3bcb8fd-d6c2-492c-ad8e-e1ab9c331423 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-2138e} kind: Pod name: ensemble-graph-2138e-94f4bbc69-5zxlj namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: fe976bc8-2285-426e-9c70-ff4c7e869605 kind: Event lastTimestamp: "2026-05-26T16:36:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:23Z" name: ensemble-graph-2138e-94f4bbc69-5zxlj.18b32ad1eccb42d2 namespace: kserve-ci-e2e-test resourceVersion: "26319" uid: e63212ef-e36c-4431-a497-11be20ac879a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-2138e-94f4bbc69 namespace: kserve-ci-e2e-test resourceVersion: "22168" uid: 4d71b11e-fbfb-49ca-80a6-b290533130fd kind: Event lastTimestamp: "2026-05-26T16:27:41Z" message: 'Created pod: ensemble-graph-2138e-94f4bbc69-5zxlj' metadata: creationTimestamp: "2026-05-26T16:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:41Z" name: ensemble-graph-2138e-94f4bbc69.18b32a5e1df9fbeb namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: 74e115f6-3e21-4a43-a9a0-81b0c8287b41 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-2138e namespace: kserve-ci-e2e-test resourceVersion: "22166" uid: 8a204bb5-af74-4e99-b006-dcde7c631732 kind: Event lastTimestamp: "2026-05-26T16:27:41Z" message: Scaled up replica set ensemble-graph-2138e-94f4bbc69 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:41Z" name: ensemble-graph-2138e.18b32a5e1d5f0ceb namespace: kserve-ci-e2e-test resourceVersion: "22169" uid: 34d89b24-6373-4d36-bb24-ab274452016a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-2138e namespace: kserve-ci-e2e-test resourceVersion: "22182" uid: 000c66da-4af4-4b19-9f6f-33d09155aba0 kind: Event lastTimestamp: "2026-05-26T16:27:41Z" message: InferenceGraph [ensemble-graph-2138e] is Ready metadata: creationTimestamp: "2026-05-26T16:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:41Z" name: ensemble-graph-2138e.18b32a5e20e0933c namespace: kserve-ci-e2e-test resourceVersion: "22184" uid: 656d18a4-b7ab-4011-8dd6-971cee280424 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-2138e namespace: kserve-ci-e2e-test resourceVersion: "22156" uid: 000c66da-4af4-4b19-9f6f-33d09155aba0 kind: Event lastTimestamp: "2026-05-26T16:27:41Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-2138e": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-2138e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:41Z" name: ensemble-graph-2138e.18b32a5e2319d320 namespace: kserve-ci-e2e-test resourceVersion: "22189" uid: 88947ec3-76f3-46a4-bb88-e3b72f72b7e8 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-2138e namespace: kserve-ci-e2e-test resourceVersion: "22156" uid: 000c66da-4af4-4b19-9f6f-33d09155aba0 kind: Event lastTimestamp: "2026-05-26T16:27:41Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-2138e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:41Z" name: ensemble-graph-2138e.18b32a5e231a30bb namespace: kserve-ci-e2e-test resourceVersion: "22190" uid: 73eff5c2-dca7-416b-97d7-a167f799e1e2 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:27:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-2138e namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: 96389d12-de1e-4580-b97a-06ab0ccc08fc kind: Event lastTimestamp: "2026-05-26T16:28:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:11Z" name: ensemble-graph-2138e.18b32a619bcb4162 namespace: kserve-ci-e2e-test resourceVersion: "22454" uid: 894057c8-0d44-4246-8fc0-4a4bd98a7b1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:27:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-2138e namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: 96389d12-de1e-4580-b97a-06ab0ccc08fc kind: Event lastTimestamp: "2026-05-26T16:28:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:11Z" name: ensemble-graph-2138e.18b32a619c2d2c37 namespace: kserve-ci-e2e-test resourceVersion: "22455" uid: 05a5545f-983a-4405-876b-5cf34f8163d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:25:31.287456Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a3fcf1b65e4 namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: df1572e8-ccd7-4eb2-b880-8020aac9cb35 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a3fe84f2ae5 namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: 36ebb8e7-d4cb-4aa1-b360-760691871ede reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a3fe9a058fb namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 8bbd09ca-3211-47f7-9b9b-d09b129c9185 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a3fed2cef59 namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: fbd9496d-b12d-4c28-b592-ba2a0115b97e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a3fed9f3ea5 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 31db3d79-ef01-4140-9087-470707d7d330 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a3fedab9ac0 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 47c3d262-c873-4631-a094-a4402ea5439f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a3ff0b55692 namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 625a7e10-865d-4923-a874-069933e748b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a3ff1260db5 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: ccaac022-425d-43ff-a53b-a3e8328d73f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:25:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:26:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:09Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a4035f9b4d7 namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: 183ff7a4-3f44-4383-96c1-c0791d01e79e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:41Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a501d380c55 namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: 8255ce8a-e3ea-4389-9f85-65a2326944c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:41Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a501d38b65c namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 8932320e-d2a9-4e17-b934-e1d7fdbd60b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: b0c00405-6776-4b04-8d3a-7dc4d309da35 kind: Event lastTimestamp: "2026-05-26T16:26:44Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:44Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25.18b32a50be0c7efd namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: e8b41ac0-5e9f-4183-aed3-b510fac3be6f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-007a5-predictor-d4b6c4bd7 namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 3e9f040c-6ed9-4e19-b2dc-bfcf285cac09 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: 'Created pod: error-404-isvc-007a5-predictor-d4b6c4bd7-9hz25' metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor-d4b6c4bd7.18b32a3fce9fbbce namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 7a86bb23-b9b0-4aaf-9a7b-f5d439fd1271 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: 28bc63b0-759e-43e2-8a5f-cbcaf195b9ea kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Scaled up replica set error-404-isvc-007a5-predictor-d4b6c4bd7 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5-predictor.18b32a3fccec123b namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: c29cc00e-4a76-41da-89d4-996f1d653532 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: 0ce97ed8-0af5-4f34-a06b-282ba285a788 kind: Event lastTimestamp: "2026-05-26T16:26:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:01Z" name: error-404-isvc-007a5-predictor.18b32a434bd73dc1 namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: 83305d02-8af4-4e09-ad2c-d525e5cb492a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: 0ce97ed8-0af5-4f34-a06b-282ba285a788 kind: Event lastTimestamp: "2026-05-26T16:26:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:01Z" name: error-404-isvc-007a5-predictor.18b32a434c57d7b8 namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 0526cfd6-5876-4195-9e87-2cf071fe3412 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20666" uid: 0ce97ed8-0af5-4f34-a06b-282ba285a788 kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:31Z" name: error-404-isvc-007a5-predictor.18b32a4a49ae7ef9 namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: 47c508f3-32fe-4ac6-b816-75fa01e15b4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20666" uid: 0ce97ed8-0af5-4f34-a06b-282ba285a788 kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:31Z" name: error-404-isvc-007a5-predictor.18b32a4a4a0eb1cd namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 27bd851e-dd8d-4f5c-827f-fca80245f7d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-007a5 namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: dbfd6d35-7e0f-4204-9d09-c18d94509175 kind: Event lastTimestamp: "2026-05-26T16:26:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:19Z" name: error-404-isvc-007a5.18b32a3fcd1fdb94 namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: 3c35b14b-b7cc-4c0b-bdd8-d48d79d8c0da reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-007a5 namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: dbfd6d35-7e0f-4204-9d09-c18d94509175 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-007a5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-007a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5.18b32a3fcda5927e namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: e10a3500-66ae-4553-a237-fde7d282fb6e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-007a5 namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: dbfd6d35-7e0f-4204-9d09-c18d94509175 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-007a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-007a5.18b32a3fcda5d933 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 798831b7-5e9b-48bf-9c37-9dc71c7d2ccb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-007a5 namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: dbfd6d35-7e0f-4204-9d09-c18d94509175 kind: Event lastTimestamp: "2026-05-26T16:26:19Z" message: InferenceService [error-404-isvc-007a5] is Ready metadata: creationTimestamp: "2026-05-26T16:26:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:19Z" name: error-404-isvc-007a5.18b32a4af09c9673 namespace: kserve-ci-e2e-test resourceVersion: "21070" uid: dc3358c5-8a34-4304-940b-92491485dc17 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:26:41.496632Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2138e-predictor-f5c8b557f-z8xts to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:26:41Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a5027e66dfc namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: caef2079-0939-42f1-b47e-b0d7ac0ef3df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-2138e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:41Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a50318ec6f8 namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: 789a3fca-6e6b-4ae4-a2a3-0faf95c8a996 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:26:42Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a5064f62025 namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: ab3784d7-4424-42ca-866d-0c587408c9f6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a5066cc600a namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 402baa5b-07e4-49df-a8ce-fca355ad0de6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a506a587e97 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 1a787c7f-619c-4505-b96d-eb64bc39516e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a506acad0db namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 57b57c75-8626-4264-9bae-90d31f1597a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a506ad7d686 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: ac86e60d-bfaa-49a5-9c23-a01e394afabc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a506d6f2b0b namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 7ec719b8-da75-4d22-bdde-1659fd3ed77d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a506ddc2c46 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 3ef6515f-73d9-4848-9f10-c7cf038c1cc4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:27:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:20Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32a50ca2259b2 namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: e1b15009-94d4-4156-9f81-0612cdb7ddb4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32ad1574660b9 namespace: kserve-ci-e2e-test resourceVersion: "26037" uid: c39696fb-3844-4933-909b-69efb4662783 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7339f829-8832-48c6-ab59-28c69db0f320 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-2138e-predictor-f5c8b557f-z8xts.18b32ad15746ff42 namespace: kserve-ci-e2e-test resourceVersion: "26038" uid: c46cb4d3-cf10-4c22-bbe9-148c2e8eed6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2138e-predictor-f5c8b557f namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7c5a3f48-b893-41ef-8dd0-4925726d6216 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: 'Created pod: error-404-isvc-2138e-predictor-f5c8b557f-z8xts' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:41Z" name: error-404-isvc-2138e-predictor-f5c8b557f.18b32a50275b428c namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 0a87d9c0-22a6-4e75-94ff-e6e6eb401c52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: 69bafc65-7103-4d48-87cc-f040b03b2ad3 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: Scaled up replica set error-404-isvc-2138e-predictor-f5c8b557f from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:41Z" name: error-404-isvc-2138e-predictor.18b32a5025a34c73 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 2a282740-8bb8-4926-a363-925bf71659de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 66ee3e79-69d3-4779-9131-2705537f4e56 kind: Event lastTimestamp: "2026-05-26T16:27:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:41Z" name: error-404-isvc-2138e-predictor.18b32a53a47ed14f namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: f5720cf8-414c-469a-b0a9-1cbce7f26604 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 66ee3e79-69d3-4779-9131-2705537f4e56 kind: Event lastTimestamp: "2026-05-26T16:27:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:41Z" name: error-404-isvc-2138e-predictor.18b32a53a4e95aba namespace: kserve-ci-e2e-test resourceVersion: "22187" uid: 2c9b0905-4efd-4a69-afbb-eb76db8a0284 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: 66ee3e79-69d3-4779-9131-2705537f4e56 kind: Event lastTimestamp: "2026-05-26T16:27:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:26Z" name: error-404-isvc-2138e-predictor.18b32a57238d73d1 namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: 4b55df61-66e0-45cc-8259-4887adf99e9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: 66ee3e79-69d3-4779-9131-2705537f4e56 kind: Event lastTimestamp: "2026-05-26T16:27:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:26Z" name: error-404-isvc-2138e-predictor.18b32a5723f507d3 namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: 6e3b0b70-42cf-427a-b76b-a77b91e78308 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2138e namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 7ba283e5-8824-406e-b86a-73a18b3ba7e4 kind: Event lastTimestamp: "2026-05-26T16:27:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:30Z" name: error-404-isvc-2138e.18b32a5025d9435b namespace: kserve-ci-e2e-test resourceVersion: "22069" uid: 25da3950-c531-4a6c-8d4f-c37199fb61fb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2138e namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 7ba283e5-8824-406e-b86a-73a18b3ba7e4 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-2138e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2138e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:41Z" name: error-404-isvc-2138e.18b32a5026463cd2 namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 76559b86-9476-409a-a615-a4e00cc1ce04 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2138e namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 7ba283e5-8824-406e-b86a-73a18b3ba7e4 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2138e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:41Z" name: error-404-isvc-2138e.18b32a502646758f namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: 0a102419-b797-4b89-a2cb-a5782abfda80 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2138e namespace: kserve-ci-e2e-test resourceVersion: "22067" uid: 7ba283e5-8824-406e-b86a-73a18b3ba7e4 kind: Event lastTimestamp: "2026-05-26T16:27:30Z" message: InferenceService [error-404-isvc-2138e] is Ready metadata: creationTimestamp: "2026-05-26T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:30Z" name: error-404-isvc-2138e.18b32a5b84e9d35f namespace: kserve-ci-e2e-test resourceVersion: "22068" uid: b742a861-c784-4a95-9c49-70816e751532 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:37:06.700460Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26962" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:37:06Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32ae1b8f3cc58 namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: 702ee673-e1b7-4e65-bbd5-2e8c9e376048 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26974" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:37:07Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32ae1d2708c07 namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: a56b8322-8a56-4d88-b2d8-8813c02dcb87 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32ae1d3bafc2e namespace: kserve-ci-e2e-test resourceVersion: "26991" uid: 03890152-a92b-4e36-9d0f-6a7df11e80af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32ae1d6c30790 namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: dd949806-3243-4927-b931-0e3e9102448e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32ae1d736b55e namespace: kserve-ci-e2e-test resourceVersion: "26996" uid: d88a9fc1-eff4-49ea-a04c-0e230d67f502 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32ae1d742efa5 namespace: kserve-ci-e2e-test resourceVersion: "26997" uid: 49581b94-7034-4193-ade4-88ec5b09ac37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32ae1dace3556 namespace: kserve-ci-e2e-test resourceVersion: "26999" uid: b7536f86-afbb-4026-9efe-3363a3868d83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32ae1db79c7f7 namespace: kserve-ci-e2e-test resourceVersion: "27000" uid: 9fc495ec-02f4-4a31-96f1-69da14c14fff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:37:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:37:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:37:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:44Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32ae23bebbebd namespace: kserve-ci-e2e-test resourceVersion: "27371" uid: f3a1cddc-547a-4891-97c6-c7c9047cbc28 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:46:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:46:21Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32b62e2452cbc namespace: kserve-ci-e2e-test resourceVersion: "31060" uid: c40b68f2-f7c0-4dfe-b2a6-b4f65a6eb622 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: b7950198-cec3-4406-8d62-6befd91ffa5d kind: Event lastTimestamp: "2026-05-26T16:46:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:46:21Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9.18b32b62e245d342 namespace: kserve-ci-e2e-test resourceVersion: "31061" uid: 93813bea-9d72-4566-a108-aee493ec4d96 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5798a-predictor-7c9d6c8d5c namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: b874066b-e159-441a-a798-e3c2304fd508 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'Created pod: error-404-isvc-5798a-predictor-7c9d6c8d5c-xjxw9' metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:06Z" name: error-404-isvc-5798a-predictor-7c9d6c8d5c.18b32ae1b85aa384 namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 7acf7ece-b568-456a-960c-bb3bb6f75658 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26953" uid: 66bd0c60-8ffb-48af-a324-ee79591733ff kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: Scaled up replica set error-404-isvc-5798a-predictor-7c9d6c8d5c from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:06Z" name: error-404-isvc-5798a-predictor.18b32ae1b6aaed55 namespace: kserve-ci-e2e-test resourceVersion: "26957" uid: a454466d-accf-451e-83d2-283db31c9dde reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26954" uid: 691d6a1a-4b2e-4572-ab44-1ce4b986a4e1 kind: Event lastTimestamp: "2026-05-26T16:37:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:51Z" name: error-404-isvc-5798a-predictor.18b32ae5350d1a20 namespace: kserve-ci-e2e-test resourceVersion: "27431" uid: 9800b41f-75ad-4cca-b490-f3833f3d56a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26954" uid: 691d6a1a-4b2e-4572-ab44-1ce4b986a4e1 kind: Event lastTimestamp: "2026-05-26T16:37:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:51Z" name: error-404-isvc-5798a-predictor.18b32ae535640e49 namespace: kserve-ci-e2e-test resourceVersion: "27432" uid: 51c15566-64d6-4c46-a327-8644aff3d5d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27136" uid: 691d6a1a-4b2e-4572-ab44-1ce4b986a4e1 kind: Event lastTimestamp: "2026-05-26T16:38:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:21Z" name: error-404-isvc-5798a-predictor.18b32ae8b40b1809 namespace: kserve-ci-e2e-test resourceVersion: "27722" uid: 8a02120f-6540-4048-b3b8-5683cbf7358a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27136" uid: 691d6a1a-4b2e-4572-ab44-1ce4b986a4e1 kind: Event lastTimestamp: "2026-05-26T16:38:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:21Z" name: error-404-isvc-5798a-predictor.18b32ae8b45b5d0e namespace: kserve-ci-e2e-test resourceVersion: "27723" uid: c21cc1a9-02ac-4fd9-92a0-97276875c0e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5798a namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: 2ec61b20-c695-4c9a-9cc1-f5b2edca255f kind: Event lastTimestamp: "2026-05-26T16:37:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:55Z" name: error-404-isvc-5798a.18b32ae1b6828a85 namespace: kserve-ci-e2e-test resourceVersion: "27475" uid: 2dda0027-1692-4bbe-bebc-2071add1f228 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5798a namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: 2ec61b20-c695-4c9a-9cc1-f5b2edca255f kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-5798a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5798a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:06Z" name: error-404-isvc-5798a.18b32ae1b6fe5707 namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 579c7b6e-5513-4817-889c-9ac203aa40fc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5798a namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: 2ec61b20-c695-4c9a-9cc1-f5b2edca255f kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5798a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:06Z" name: error-404-isvc-5798a.18b32ae1b6fead21 namespace: kserve-ci-e2e-test resourceVersion: "26960" uid: ae041d28-efa8-4569-accf-1ae3f7db7b53 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5798a namespace: kserve-ci-e2e-test resourceVersion: "27473" uid: 2ec61b20-c695-4c9a-9cc1-f5b2edca255f kind: Event lastTimestamp: "2026-05-26T16:37:54Z" message: InferenceService [error-404-isvc-5798a] is Ready metadata: creationTimestamp: "2026-05-26T16:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:54Z" name: error-404-isvc-5798a.18b32aecf6ac9f2d namespace: kserve-ci-e2e-test resourceVersion: "27474" uid: d61cf8b5-d945-4697-abcc-66fd37bf8f41 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:35:56.462235Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26066" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-82fbc-predictor-855f648bc-wjldp to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ad15e6d78c4 namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: ca572e22-3378-4988-9941-784dea955ce8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26078" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ad177d3fd51 namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: 7578c887-5c78-4c37-9a66-ec774327588e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ad1792ecec5 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: 82f5f32a-888c-44a5-904a-6641eeab3ce2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ad17c70f737 namespace: kserve-ci-e2e-test resourceVersion: "26100" uid: 223e3b5a-7322-4469-92a3-4976d9b42c53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ad17ce27358 namespace: kserve-ci-e2e-test resourceVersion: "26101" uid: 51240db5-6d4a-463d-bae6-e562258744f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ad17cf0b8a7 namespace: kserve-ci-e2e-test resourceVersion: "26102" uid: 4d82203e-3a6f-475d-b073-1f45070e3706 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:35:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:57Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ad17fcb4158 namespace: kserve-ci-e2e-test resourceVersion: "26103" uid: f0feae34-3498-4545-8f42-263e0f49ae41 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:35:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:57Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ad1803a29c0 namespace: kserve-ci-e2e-test resourceVersion: "26104" uid: fff8c2fe-33ef-4a2e-9e80-bb526c2497cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:36:34Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:34Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ad1e3e48e89 namespace: kserve-ci-e2e-test resourceVersion: "26404" uid: 938daacd-1e80-44e3-ae2e-1ab312ef7bac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:06Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ae1b4f8e4f6 namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: aa117b95-0a05-4f66-9e4e-d984f93def42 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:06Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ae1b4fa0c46 namespace: kserve-ci-e2e-test resourceVersion: "26950" uid: 53cbae03-f5cf-4124-805a-5f2518c24876 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-82fbc-predictor-855f648bc-wjldp namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 75743ec9-c617-4958-a564-df9910f039a5 kind: Event lastTimestamp: "2026-05-26T16:37:09Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:09Z" name: error-404-isvc-82fbc-predictor-855f648bc-wjldp.18b32ae26c1988ba namespace: kserve-ci-e2e-test resourceVersion: "27027" uid: fa803e2e-37d8-48c1-8349-b6f010f7635d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-82fbc-predictor-855f648bc namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: b5fdc83a-6ba2-4f74-9f45-34d9e52228bd kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: 'Created pod: error-404-isvc-82fbc-predictor-855f648bc-wjldp' metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc-predictor-855f648bc.18b32ad15ddea346 namespace: kserve-ci-e2e-test resourceVersion: "26068" uid: 008f101e-3f1f-4f88-906d-b68ef5d64589 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-82fbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26055" uid: d1c84ca3-8ae0-450b-8e3e-91c859126ec8 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Scaled up replica set error-404-isvc-82fbc-predictor-855f648bc from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc-predictor.18b32ad15ba85f2e namespace: kserve-ci-e2e-test resourceVersion: "26060" uid: 5ccfd079-43f1-4c05-b1a1-1f780139f3a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-82fbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: fdc780e6-d25b-4cee-ae5a-ad1d05bacdb6 kind: Event lastTimestamp: "2026-05-26T16:36:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:56Z" name: error-404-isvc-82fbc-predictor.18b32ad4da0e58c7 namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: f56d8f8d-b54e-4d38-baeb-c9b9546c6048 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-82fbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: fdc780e6-d25b-4cee-ae5a-ad1d05bacdb6 kind: Event lastTimestamp: "2026-05-26T16:36:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:56Z" name: error-404-isvc-82fbc-predictor.18b32ad4daaf56a9 namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: ffcd5c6b-b1b8-4813-83ed-34d5685ec6a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-82fbc namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: 32eb7446-da55-4ab6-8ae5-b64ac3937268 kind: Event lastTimestamp: "2026-05-26T16:36:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:36:44Z" name: error-404-isvc-82fbc.18b32ad15b842f39 namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: e4d29ad5-dacb-4b94-98d6-a5aa53687108 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-82fbc namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: 32eb7446-da55-4ab6-8ae5-b64ac3937268 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-82fbc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-82fbc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc.18b32ad15bd054e4 namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: b36842e4-1c88-4436-9aff-21b652a7e8fa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-82fbc namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: 32eb7446-da55-4ab6-8ae5-b64ac3937268 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-82fbc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:35:56Z" name: error-404-isvc-82fbc.18b32ad15bd0b71d namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 27880ecd-58f2-4546-8696-3505d5833f6e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-82fbc namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 32eb7446-da55-4ab6-8ae5-b64ac3937268 kind: Event lastTimestamp: "2026-05-26T16:36:44Z" message: InferenceService [error-404-isvc-82fbc] is Ready metadata: creationTimestamp: "2026-05-26T16:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:36:44Z" name: error-404-isvc-82fbc.18b32adc9f173c80 namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: 4e4fa669-5e1d-49ea-b44a-24dd5409ae7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:27:21.266234Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-86373-predictor-776c7fd767-prf4x to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:27:21Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a596a5a6298 namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: 2d36f6fa-977e-469c-9d54-2d6d20df93ac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-86373-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:21Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a5970da758a namespace: kserve-ci-e2e-test resourceVersion: "21934" uid: 95d16380-8d34-4691-9f5b-b738b34d6506 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21933" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:27:21Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a59958501a2 namespace: kserve-ci-e2e-test resourceVersion: "21946" uid: aa164b31-15e8-4c78-971c-9244771580fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a5996c719cc namespace: kserve-ci-e2e-test resourceVersion: "21948" uid: 5a0517ea-195f-4556-9782-de6c61ae55b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a599a1dc0a3 namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: 4e8291a4-a33c-4eec-b56a-cd3e428d1853 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a599a906168 namespace: kserve-ci-e2e-test resourceVersion: "21953" uid: 8ca60357-40f5-4ee7-801a-1e5fb6fb6500 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a599a9d1d1c namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 4f686859-08e2-4354-8dc3-321eb2aa7923 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a599d4d5a86 namespace: kserve-ci-e2e-test resourceVersion: "21955" uid: f4d6c1b6-e6c4-4250-90a3-81450ab5110d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a599dbe6ea4 namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: 2bd55e27-9b1f-4238-82fb-c8846d3be256 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:27:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:27:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:27:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:59Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32a59e52d02c4 namespace: kserve-ci-e2e-test resourceVersion: "22347" uid: 92b2db6c-bcbb-422f-938b-5903651e5932 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:36:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:35Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32ada91fefec1 namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 9e78a7c0-1b5f-4c3c-8278-4d98005edeab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:36:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:35Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32ada91ffc412 namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: 8fcd8740-3b8d-4a8b-85d4-cc2b3488372d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-86373-predictor-776c7fd767-prf4x namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9f0269a7-24fc-4ac2-8813-52425891e08e kind: Event lastTimestamp: "2026-05-26T16:36:40Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-05-26T16:36:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:40Z" name: error-404-isvc-86373-predictor-776c7fd767-prf4x.18b32adb953b8b3a namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 4d4d7dea-733e-4e74-96bb-9bc53840153a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-86373-predictor-776c7fd767 namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: db63418b-3aa5-42db-a795-47b4d71e68f4 kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: 'Created pod: error-404-isvc-86373-predictor-776c7fd767-prf4x' metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:21Z" name: error-404-isvc-86373-predictor-776c7fd767.18b32a5969f365ee namespace: kserve-ci-e2e-test resourceVersion: "21923" uid: db6f1c62-1dbd-4448-bb44-e86a65665bd1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: f7f2dce8-2248-4feb-94c3-cb2ba176d426 kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: Scaled up replica set error-404-isvc-86373-predictor-776c7fd767 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:21Z" name: error-404-isvc-86373-predictor.18b32a59685a071d namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: d5c4f292-9566-4442-acc7-1a86b3c257f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:27:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: fcc1e22b-5c5a-4654-8069-f526a095c92f kind: Event lastTimestamp: "2026-05-26T16:28:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:06Z" name: error-404-isvc-86373-predictor.18b32a5ce6d5af85 namespace: kserve-ci-e2e-test resourceVersion: "22399" uid: 66f84c51-c0ec-477d-8567-5c0c39ba09c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:27:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: fcc1e22b-5c5a-4654-8069-f526a095c92f kind: Event lastTimestamp: "2026-05-26T16:28:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:06Z" name: error-404-isvc-86373-predictor.18b32a5ce7314ac3 namespace: kserve-ci-e2e-test resourceVersion: "22400" uid: bdc1bd66-74f7-4e52-9d1c-49fc9d92ea37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:28:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "22117" uid: fcc1e22b-5c5a-4654-8069-f526a095c92f kind: Event lastTimestamp: "2026-05-26T16:28:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:28:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:51Z" name: error-404-isvc-86373-predictor.18b32a676353d0c6 namespace: kserve-ci-e2e-test resourceVersion: "22789" uid: f72a5cdf-5e40-4490-9241-e4df16ba7ccf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:28:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "22117" uid: fcc1e22b-5c5a-4654-8069-f526a095c92f kind: Event lastTimestamp: "2026-05-26T16:28:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:28:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:51Z" name: error-404-isvc-86373-predictor.18b32a6763b0716e namespace: kserve-ci-e2e-test resourceVersion: "22790" uid: 6625302d-bb6f-43e2-973d-7fd8ab1f3410 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-86373 namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 595e50f8-3a60-4051-95f2-58045bc4007e kind: Event lastTimestamp: "2026-05-26T16:28:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:28:09Z" name: error-404-isvc-86373.18b32a59682f6084 namespace: kserve-ci-e2e-test resourceVersion: "22439" uid: ddd24117-e012-49ce-8359-4fce79a05d39 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-86373 namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 595e50f8-3a60-4051-95f2-58045bc4007e kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-86373": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-86373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:21Z" name: error-404-isvc-86373.18b32a5968a4d109 namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: 2f78a514-dc47-4e43-b1a9-b592f290386b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-86373 namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 595e50f8-3a60-4051-95f2-58045bc4007e kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-86373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:21Z" name: error-404-isvc-86373.18b32a5968a5180f namespace: kserve-ci-e2e-test resourceVersion: "21920" uid: b1965f68-4614-40a0-9c64-f429c6cf95ce reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-86373 namespace: kserve-ci-e2e-test resourceVersion: "22437" uid: 595e50f8-3a60-4051-95f2-58045bc4007e kind: Event lastTimestamp: "2026-05-26T16:28:09Z" message: InferenceService [error-404-isvc-86373] is Ready metadata: creationTimestamp: "2026-05-26T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:28:09Z" name: error-404-isvc-86373.18b32a64a0c12902 namespace: kserve-ci-e2e-test resourceVersion: "22438" uid: e3b7229c-21cf-43fd-a029-5406fef5e2c7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:26:11.038042Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a007a-predictor-c5d584875-l64qx to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:26:11Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a49106d3ad0 namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: 1da08166-d4c1-4539-b67d-b515f6ea9452 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:26:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a007a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:11Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a491c1c1423 namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: 7a8288a4-a774-452a-9a81-ce7fc424e2a0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:26:12Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a494d85d9bb namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 4e1956fa-3a34-425d-ad31-76b326cead12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a494ed529c2 namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 7d244330-468c-4c91-9835-d8dc8f9ef380 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a4951c86032 namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: a8c267b8-6313-42f8-b3bc-72cbaf1e846e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a49523c0180 namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: e1c3c165-1cf5-47e9-8c84-885ac308f540 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a4952470eb8 namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 8d77e4d4-3ebb-4366-a2dd-37dc321ab8d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a4955cf4522 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: b5c9b7dd-9ff4-4f46-aa9f-873e3fa29224 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a4956417db6 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 2dab8c5b-5a0b-4ce3-ab50-39b5759ef3c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:26:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:50Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a49c90821e1 namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: e7fc5cca-02c5-4d12-8a82-e9dd58947bc7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:21Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a5966f4bd36 namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: f6d059db-b646-4a61-afaf-18cf285096b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a007a-predictor-c5d584875-l64qx namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 8430529d-fb3d-44c3-836d-8f123b3dc2a6 kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:21Z" name: error-404-isvc-a007a-predictor-c5d584875-l64qx.18b32a5966f5680b namespace: kserve-ci-e2e-test resourceVersion: "21912" uid: e84e3903-0414-4b62-8516-a5f00ea6e8b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a007a-predictor-c5d584875 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: 66b4ebd9-833e-415b-87e1-6dc73f9a8973 kind: Event lastTimestamp: "2026-05-26T16:26:11Z" message: 'Created pod: error-404-isvc-a007a-predictor-c5d584875-l64qx' metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:11Z" name: error-404-isvc-a007a-predictor-c5d584875.18b32a490ffbd037 namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: a536851e-c301-457d-a50c-eaef77320240 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: c981bf74-ce52-4871-b490-0b30aae9a11e kind: Event lastTimestamp: "2026-05-26T16:26:11Z" message: Scaled up replica set error-404-isvc-a007a-predictor-c5d584875 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:11Z" name: error-404-isvc-a007a-predictor.18b32a490e652e17 namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 68a9941c-c598-4dc8-a604-5762a8fcaf07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: 92d216ca-1ac7-498b-8c23-87591ad5d4ef kind: Event lastTimestamp: "2026-05-26T16:26:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:56Z" name: error-404-isvc-a007a-predictor.18b32a4c8ce130af namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: 5d6cc256-f685-4797-851e-037ea71b7a25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: 92d216ca-1ac7-498b-8c23-87591ad5d4ef kind: Event lastTimestamp: "2026-05-26T16:26:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:56Z" name: error-404-isvc-a007a-predictor.18b32a4c8d40b3ce namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: 38847070-c2c8-4a51-9d4b-489b4f052563 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 92d216ca-1ac7-498b-8c23-87591ad5d4ef kind: Event lastTimestamp: "2026-05-26T16:27:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:11Z" name: error-404-isvc-a007a-predictor.18b32a500be54a2d namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 5c0132e6-6cd8-4232-a3e1-fb2ba6780518 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 92d216ca-1ac7-498b-8c23-87591ad5d4ef kind: Event lastTimestamp: "2026-05-26T16:27:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:11Z" name: error-404-isvc-a007a-predictor.18b32a500c2aebed namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 60fca04e-ac1c-4f9e-8f16-f411bdc8d9cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a007a namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 7cf14842-e9eb-40f3-b052-f08440f1422e kind: Event lastTimestamp: "2026-05-26T16:27:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:00Z" name: error-404-isvc-a007a.18b32a490e45f27f namespace: kserve-ci-e2e-test resourceVersion: "21625" uid: 7d63173d-1ee5-4d9a-9ae2-a592196de1fe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a007a namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 7cf14842-e9eb-40f3-b052-f08440f1422e kind: Event lastTimestamp: "2026-05-26T16:26:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-a007a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a007a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:11Z" name: error-404-isvc-a007a.18b32a490ea9d9d2 namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: bae00763-ce1a-499c-a256-626bed09316a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a007a namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 7cf14842-e9eb-40f3-b052-f08440f1422e kind: Event lastTimestamp: "2026-05-26T16:26:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a007a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:11Z" name: error-404-isvc-a007a.18b32a490eaa25ba namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: fbed9236-7aeb-4ac6-b77e-ca07846a4c2c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a007a namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 7cf14842-e9eb-40f3-b052-f08440f1422e kind: Event lastTimestamp: "2026-05-26T16:27:00Z" message: InferenceService [error-404-isvc-a007a] is Ready metadata: creationTimestamp: "2026-05-26T16:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:00Z" name: error-404-isvc-a007a.18b32a5483fd0a73 namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: f5d78b68-297c-4706-8df7-95f813d9caeb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:16:56.157556Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ad62d-predictor-69787655ff-l67xd to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:16:56Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c7def8e88b namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: fb43f903-c4cf-455d-a4a6-accda232e9aa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ad62d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:56Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c7eb22cc26 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: 6ae46512-de76-45db-ab38-f8908e4ff212 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:16:57Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c81c44379e namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: 9db9d64e-b839-4fbe-8cfa-de93c940a1ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c81da90793 namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 1ded19f8-0140-4dc7-b635-88b9236d9e15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c8204e7a1d namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: e9313eac-44ca-4091-aaf1-429dbb23b47d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c820be01a6 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: c8c630d3-75ae-4fdd-aad8-9694aa931d1b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c820c94eaa namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 5f52c576-6f0b-4740-9108-25e3e5c665b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c82397afce namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: b122d27b-1399-43e8-a0ff-42a912f42d0b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c8240501c5 namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: 0f41d9d1-c8b5-4219-8554-75a0be20766d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:26:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:13Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b329c8370fb6ee namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: d4ccec3a-3503-45ea-8222-bc333786f0d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:26:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:11Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b32a490e20e33a namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: bbb74a86-3daa-43b3-985f-43db0eb513be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:26:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:11Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b32a490e21709a namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: 7a831780-8b50-4c73-aab2-1a6a7f0d11ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad62d-predictor-69787655ff-l67xd namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: e8cc1d7e-b614-43f6-acbf-7d42248fbbe0 kind: Event lastTimestamp: "2026-05-26T16:26:13Z" message: 'Readiness probe failed: Get "https://10.132.0.21:8643/healthz": dial tcp 10.132.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:13Z" name: error-404-isvc-ad62d-predictor-69787655ff-l67xd.18b32a49ab69b60a namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: 3c5fa83f-f8ba-4bf4-8992-fd0fbd04a4bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ad62d-predictor-69787655ff namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 1067122a-7b54-4c52-8439-632682463bad kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: 'Created pod: error-404-isvc-ad62d-predictor-69787655ff-l67xd' metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:56Z" name: error-404-isvc-ad62d-predictor-69787655ff.18b329c7de805f43 namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: 10e52781-cd38-4836-9219-8dfded24f1a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 8f02230b-692d-4076-9db7-8863d649c517 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: Scaled up replica set error-404-isvc-ad62d-predictor-69787655ff from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:56Z" name: error-404-isvc-ad62d-predictor.18b329c7dcf2f28a namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: 8a185054-e1d0-42b9-aa54-2b34c90b385e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 0cbc4cc4-9203-452d-898f-993b2f33ca55 kind: Event lastTimestamp: "2026-05-26T16:17:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:41Z" name: error-404-isvc-ad62d-predictor.18b329cb5b72b4d8 namespace: kserve-ci-e2e-test resourceVersion: "17151" uid: 5bc82467-421a-485a-98da-fe889b929c1f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 0cbc4cc4-9203-452d-898f-993b2f33ca55 kind: Event lastTimestamp: "2026-05-26T16:17:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:41Z" name: error-404-isvc-ad62d-predictor.18b329cb5bd45dbc namespace: kserve-ci-e2e-test resourceVersion: "17152" uid: fa0172dc-f2ae-474c-8966-5b46d61fa7d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: 0cbc4cc4-9203-452d-898f-993b2f33ca55 kind: Event lastTimestamp: "2026-05-26T16:18:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:26Z" name: error-404-isvc-ad62d-predictor.18b329d5d7f79a83 namespace: kserve-ci-e2e-test resourceVersion: "17540" uid: 7ef031bf-1715-465f-a835-ab026dd844e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: 0cbc4cc4-9203-452d-898f-993b2f33ca55 kind: Event lastTimestamp: "2026-05-26T16:18:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:26Z" name: error-404-isvc-ad62d-predictor.18b329d5d85a380f namespace: kserve-ci-e2e-test resourceVersion: "17541" uid: 9622a11d-b64f-45a5-bd01-b44f3fd22ad6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ad62d namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: e2081a22-5952-4344-8ee0-12e26ace74e3 kind: Event lastTimestamp: "2026-05-26T16:17:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:43Z" name: error-404-isvc-ad62d.18b329c7dcd37d6a namespace: kserve-ci-e2e-test resourceVersion: "17175" uid: 4c75a05f-e66b-4319-a397-b4f9756ffb31 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ad62d namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: e2081a22-5952-4344-8ee0-12e26ace74e3 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-ad62d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ad62d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:56Z" name: error-404-isvc-ad62d.18b329c7dd690ba4 namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: 19af2ebd-e3c8-4847-86b3-9f431e9ccd56 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ad62d namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: e2081a22-5952-4344-8ee0-12e26ace74e3 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ad62d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:56Z" name: error-404-isvc-ad62d.18b329c7dd695e96 namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: 2da95734-df70-4359-8ce1-ca3df726cd8b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ad62d namespace: kserve-ci-e2e-test resourceVersion: "17173" uid: e2081a22-5952-4344-8ee0-12e26ace74e3 kind: Event lastTimestamp: "2026-05-26T16:17:43Z" message: InferenceService [error-404-isvc-ad62d] is Ready metadata: creationTimestamp: "2026-05-26T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:43Z" name: error-404-isvc-ad62d.18b329d2f19c79a8 namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: 4b163ecc-4dfd-4568-a0b8-f268e535e2a3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:36:36.200046Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26508" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32ada9efc632e namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: 360d1b42-f06b-4825-ac83-a397809e51d7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32adab85763d5 namespace: kserve-ci-e2e-test resourceVersion: "26531" uid: 10aba533-472c-4fc9-ac35-005a082edd70 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32adab98fc6bd namespace: kserve-ci-e2e-test resourceVersion: "26535" uid: 89b442d6-05f4-48f4-a1c2-74ba87db94f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32adabc3bbd94 namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: 5cc43efd-0b0a-412a-8c39-35fa0db416bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32adabcab7f49 namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: 0149446a-ad14-4481-ad8b-7f057cb2f1b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32adabcb83a71 namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: cca8a31a-c69a-4c9c-b165-e1a7a2802299 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32adabf96b778 namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: 771b64d5-407a-4fde-9376-926892fca366 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32adac004385c namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: e6f914a5-efe9-4e39-b1af-fe521962c529 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:37:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:13Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32adaff14c568 namespace: kserve-ci-e2e-test resourceVersion: "27061" uid: 48ddc3d1-6589-427a-b652-8bdee6236d13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:53:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:53:55Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32bcc972d0279 namespace: kserve-ci-e2e-test resourceVersion: "34190" uid: 1498941f-7306-4b61-87a1-235b559ddcef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ecb34959-d52b-4733-a39c-f7df77c836f0 kind: Event lastTimestamp: "2026-05-26T16:53:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:53:55Z" name: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5.18b32bcc972d8050 namespace: kserve-ci-e2e-test resourceVersion: "34191" uid: 8e7fce32-ce4d-4287-810b-a2ccf1bdfcfc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bffff-predictor-6c4898b5bf namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 38993d8f-b8bc-4f3d-97ed-4e25e96a63e0 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: 'Created pod: error-404-isvc-bffff-predictor-6c4898b5bf-4fsw5' metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor-6c4898b5bf.18b32ada9e6b6edd namespace: kserve-ci-e2e-test resourceVersion: "26509" uid: 7592bfd4-aa58-4d56-8381-1d93bf7b89f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 839dd18c-cad7-4f49-8517-e49ab9a28317 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: Scaled up replica set error-404-isvc-bffff-predictor-6c4898b5bf from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff-predictor.18b32ada9cb239ea namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: 94ab89d4-d376-4a1f-a5f7-ca9f6de6166c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:36:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 1ee374fe-7c67-428f-a8e5-5764ea53c7c6 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:06Z" name: error-404-isvc-bffff-predictor.18b32ade1b44bcd7 namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: 03bb6958-08d5-46d8-a95b-f43a1f006a15 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:36:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 1ee374fe-7c67-428f-a8e5-5764ea53c7c6 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:06Z" name: error-404-isvc-bffff-predictor.18b32ade1ba102fc namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: 0fe56a3c-3a95-4d5d-b5b6-9c948b67012d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 1ee374fe-7c67-428f-a8e5-5764ea53c7c6 kind: Event lastTimestamp: "2026-05-26T16:37:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:51Z" name: error-404-isvc-bffff-predictor.18b32ae5190fc487 namespace: kserve-ci-e2e-test resourceVersion: "27424" uid: 0285ffb1-8328-45c6-b385-a8006740da2d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 1ee374fe-7c67-428f-a8e5-5764ea53c7c6 kind: Event lastTimestamp: "2026-05-26T16:37:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:51Z" name: error-404-isvc-bffff-predictor.18b32ae519701500 namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: feebca2e-9f01-4784-8d8f-ce7b0561687d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bffff namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 53ef32f5-a743-4fba-8aea-a9c8cb128d49 kind: Event lastTimestamp: "2026-05-26T16:37:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:23Z" name: error-404-isvc-bffff.18b32ada9cad8a05 namespace: kserve-ci-e2e-test resourceVersion: "27169" uid: 741f3c41-fce5-419c-a6d2-33e2e4adca90 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bffff namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 53ef32f5-a743-4fba-8aea-a9c8cb128d49 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-bffff": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bffff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff.18b32ada9d317deb namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: 2dda03f4-e377-48a3-9152-6a6160a0b9ea reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bffff namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 53ef32f5-a743-4fba-8aea-a9c8cb128d49 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bffff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:36:36Z" name: error-404-isvc-bffff.18b32ada9d31c606 namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: a62a35d6-01fb-4cb1-901e-51c299660f70 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bffff namespace: kserve-ci-e2e-test resourceVersion: "27166" uid: 53ef32f5-a743-4fba-8aea-a9c8cb128d49 kind: Event lastTimestamp: "2026-05-26T16:37:23Z" message: InferenceService [error-404-isvc-bffff] is Ready metadata: creationTimestamp: "2026-05-26T16:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:23Z" name: error-404-isvc-bffff.18b32ae5bb1995ef namespace: kserve-ci-e2e-test resourceVersion: "27167" uid: 5b251f8f-07a6-40ec-aef9-c71dbf972b0b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:14:46.059564Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:14:46Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329a994871b49 namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 17cdd735-a7b3-4a2c-9d44-d993359d480c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c73f1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:46Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329a99f31b8f8 namespace: kserve-ci-e2e-test resourceVersion: "15080" uid: 7d5c3bb8-7f1a-447d-bcf2-4d87bdd91af2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15059" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:14:47Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:14:47Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329a9d19d0bb7 namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 1dcca0c4-d8b8-4ede-a642-8244a3e42b17 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:14:47Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T16:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:47Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329a9d35a575d namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: 4a48b2cb-e4e7-47db-8b4c-ce525333ab10 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:14:58Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" in 11.657s (11.657s including waiting). Image size: 1334799468 bytes.' metadata: creationTimestamp: "2026-05-26T16:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:58Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329ac8a2ea9d8 namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: 48df4597-fd1b-4284-bacf-f01ae71b137b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:14:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:58Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329ac8df7f526 namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 49121843-4ffb-4fd7-9480-41020148dadc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:14:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:58Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329ac8eabee3a namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 1a5551aa-6dda-443a-83ed-974c5f66d279 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:14:58Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T16:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:58Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329ac8ebe63cf namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: 293eb278-cfee-4c3c-898b-da84fceb818b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:15:02Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.412s (3.412s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T16:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:02Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329ad5a2756aa namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 1dc56e17-b76f-44b2-b28d-9fd719ab257b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:15:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:02Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329ad5e12f8c5 namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 0be2217b-688b-484b-ab71-805cd3749979 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:15:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:02Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329ad5e84c328 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: 76a5586f-a516-44ca-b2dd-5ef0509270af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:15:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:40Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329add44ac1e8 namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 0709ddb9-ddf8-4e42-a04d-c9dd1765b7e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329be89701c77 namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: 64a06112-7e2a-4ee6-9ffd-784256bb45ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 4751ea58-7db7-4b7f-b5c9-57e72c3fe07d kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7.18b329be89708ecf namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 7ac1c451-d49b-4d4b-aca4-0c3bfe3e9abe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c73f1-predictor-7c8d9dc697 namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 48b1bfaa-eba8-49f2-be50-491df07c7cba kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: 'Created pod: error-404-isvc-c73f1-predictor-7c8d9dc697-5sqs7' metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:46Z" name: error-404-isvc-c73f1-predictor-7c8d9dc697.18b329a994212e74 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 0ce38064-c858-42c2-981b-616e1102a1c3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: cd10dcb4-926a-4d8b-b68b-4f3979294140 kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: Scaled up replica set error-404-isvc-c73f1-predictor-7c8d9dc697 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:46Z" name: error-404-isvc-c73f1-predictor.18b329a99185835f namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 264d5c1c-4cbf-4379-8f8a-72e2a54e9932 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 9fede182-672a-4c9c-8e77-41f1fe01a5bf kind: Event lastTimestamp: "2026-05-26T16:15:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:01Z" name: error-404-isvc-c73f1-predictor.18b329ad0ff1b76d namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: cec59d63-2ca7-4ab4-b3f9-7468d6dddc23 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 9fede182-672a-4c9c-8e77-41f1fe01a5bf kind: Event lastTimestamp: "2026-05-26T16:15:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:01Z" name: error-404-isvc-c73f1-predictor.18b329ad106b0676 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: 7774b2fc-d3df-4a6d-914b-4605a7db8747 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 9fede182-672a-4c9c-8e77-41f1fe01a5bf kind: Event lastTimestamp: "2026-05-26T16:16:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:01Z" name: error-404-isvc-c73f1-predictor.18b329b08f0d78b3 namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: b6143e52-98f9-4675-baa1-257431fc482d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:15:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 9fede182-672a-4c9c-8e77-41f1fe01a5bf kind: Event lastTimestamp: "2026-05-26T16:16:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:01Z" name: error-404-isvc-c73f1-predictor.18b329b08f733695 namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: 6bfa98d5-f847-4d88-a1e4-a4e9a5cadeea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c73f1 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 1d00202b-00aa-4ff1-8902-e375dbe4613d kind: Event lastTimestamp: "2026-05-26T16:15:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:15:50Z" name: error-404-isvc-c73f1.18b329a9915e88cf namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 25c44bd3-942d-4f98-a955-ace69f273b38 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c73f1 namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: 1d00202b-00aa-4ff1-8902-e375dbe4613d kind: Event lastTimestamp: "2026-05-26T16:15:50Z" message: InferenceService [error-404-isvc-c73f1] is Ready metadata: creationTimestamp: "2026-05-26T16:15:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:15:50Z" name: error-404-isvc-c73f1.18b329b88f3d11b1 namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: a3744db6-a6f4-4aec-bd2e-53ba7733ad56 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:16:16.249249Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16113" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-df0d2-predictor-66496f47bf-q49z8 to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:16:16Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329be9440731c namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: d5bdd9d1-65f6-4ceb-8514-e8a7a403f4bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-df0d2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329bea02c69b8 namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 97ffb4d1-2f20-43d5-aafb-e879297d5ea8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:16:17Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329bed160701d namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: ae27eb1d-fc9e-420e-af10-681913e6b123 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:17Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329bed2e3a020 namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 9d0a1f7b-a6d5-462b-959a-3c61af2b780b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:17Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329bed5fb6cc6 namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 2125b994-6679-48f3-9610-357186d9bdbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:17Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329bed669f20b namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: cb911097-7a86-4d93-b572-41e9f6f8d6c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:17Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329bed675799f namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 74b56c1f-c76b-4c96-8383-fe0db86df99e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:17Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329bed9530355 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: a749ed7f-925e-4968-a22e-d38045cce749 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:17Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329bed9c2e864 namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: 519c685d-6732-4aa6-b87c-cd96ef3e7bf3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:16:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:16:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:16:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:54Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b329bf1b7b5318 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 80cf71a9-3fed-4fe7-8e9d-f492309e4c81 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b32a3fc9e23f9f namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 80751a19-2f9c-4abe-ae0d-c10d5ae6e571 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2fd1085c-0886-4d4d-b4b6-eeca8be643eb kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: error-404-isvc-df0d2-predictor-66496f47bf-q49z8.18b32a3fc9e34627 namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: d8e54593-4d3a-45f0-b310-a30614fa5000 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-df0d2-predictor-66496f47bf namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: 60dde12b-4b06-4f68-bf48-3aa2016732dc kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: 'Created pod: error-404-isvc-df0d2-predictor-66496f47bf-q49z8' metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:16Z" name: error-404-isvc-df0d2-predictor-66496f47bf.18b329be937b6b38 namespace: kserve-ci-e2e-test resourceVersion: "16114" uid: 818ecb74-fcf3-4a19-846e-701b2d4b91d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: db522c0b-f304-4a92-bd87-1b8dd7bd7c76 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Scaled up replica set error-404-isvc-df0d2-predictor-66496f47bf from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:16Z" name: error-404-isvc-df0d2-predictor.18b329be921e0d3d namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: eec8c977-1521-49bc-afe4-590a4bd6a1b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 82e05108-80ae-4d75-bf28-3ce1266d8285 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:46Z" name: error-404-isvc-df0d2-predictor.18b329c2108719b0 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: 75c60d13-040f-4fa5-b094-61df383d31e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 82e05108-80ae-4d75-bf28-3ce1266d8285 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:46Z" name: error-404-isvc-df0d2-predictor.18b329c210f0ce3e namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: 049567ff-eac6-49aa-acd9-8c73d639d8ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 82e05108-80ae-4d75-bf28-3ce1266d8285 kind: Event lastTimestamp: "2026-05-26T16:17:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:31Z" name: error-404-isvc-df0d2-predictor.18b329c90e3ed8df namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: 9aedb9d3-450b-4076-936d-faa523d08b9f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 82e05108-80ae-4d75-bf28-3ce1266d8285 kind: Event lastTimestamp: "2026-05-26T16:17:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:31Z" name: error-404-isvc-df0d2-predictor.18b329c90ea223b3 namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: 22dadff0-89cc-4166-bd19-4fd5cf8cf58d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 593d0e5f-cf03-441d-aacd-c6d26781552d kind: Event lastTimestamp: "2026-05-26T16:17:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:04Z" name: error-404-isvc-df0d2.18b329be91fd7b04 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: 363b6e7e-9b43-4112-90f3-6dfd817cb260 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: 593d0e5f-cf03-441d-aacd-c6d26781552d kind: Event lastTimestamp: "2026-05-26T16:17:04Z" message: InferenceService [error-404-isvc-df0d2] is Ready metadata: creationTimestamp: "2026-05-26T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:04Z" name: error-404-isvc-df0d2.18b329c9d6510aee namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: e106938c-024f-4aa1-b6f9-170440cf1315 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:14:45.969305Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-678784f46b-gshlx to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:14:45Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329a98f25e6fc namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 1dcebb0b-a77b-459d-b20f-f3fbdf95e8b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:14:46Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329a9a8dd1780 namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: bf58eb5d-965e-40e6-b97a-85bc081d758a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:46Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329a9ab24e1fd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: e8a1b24d-c13b-4594-9c3b-afe9aad2c863 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:14:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" in 3.628s (3.628s including waiting). Image size: 301466548 bytes.' metadata: creationTimestamp: "2026-05-26T16:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:50Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329aa836847b3 namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: 2115426b-957c-4281-a9e5-bafdad834caa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:14:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:50Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329aa873918ed namespace: kserve-ci-e2e-test resourceVersion: "15184" uid: 6498b677-27b3-471d-9efc-2083acf05164 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:14:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:50Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329aa87c2bd5e namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: edbdba77-0cf4-4f31-aa37-f76543afec15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:14:54Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T16:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:54Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329ab8efd3bd2 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: cb4a90df-1ec8-4455-96b8-9ec252a80778 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:15:08Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 14.377s (14.377s including waiting). Image size: 1560933300 bytes.' metadata: creationTimestamp: "2026-05-26T16:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:08Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329aee7f421be namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 693fb18d-3c13-4eda-9887-f3decb59f556 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:15:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:08Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329aeeba30898 namespace: kserve-ci-e2e-test resourceVersion: "15389" uid: 15caba43-0adb-4d7c-b3ab-40172187a2e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:15:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:15:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:09Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329aeec0ad28c namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: f1083909-387c-4c9d-aa46-c5e2b8aa9f95 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:15:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T16:15:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:09Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329aeec180cc2 namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: 387dee1f-9bd6-49b6-a584-eff1e7164afc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:15:11Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.521s (2.521s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:11Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329af826427c6 namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: d5067e2f-34e5-4bbb-9bba-5d7866617ceb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:15:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:11Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329af8620a149 namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 4d962ca4-2e5d-43e1-9bc3-7155c4c785ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:15:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:11Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329af86872e73 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 0f899fc4-0b02-49d3-ba06-6e7416ad4e77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329af88c43729 namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: 3752a4fc-0007-47c4-ad70-a3cba33190ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:56Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329c7d9645743 namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: 43522a87-b36d-4a09-8601-329448cdbeb1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:56Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329c7d964dcbd namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: 07b89504-fed2-4c5a-bbb8-825574e548c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f9a3c334-6524-46b8-b538-59f5a8327a1a kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: isvc-sklearn-graph-1-predictor-678784f46b-gshlx.18b329c837138408 namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: bc4ce225-ecd2-4b8c-82c1-87de194f9ba7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-678784f46b namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: fb25d6d5-3026-42cd-ba8b-b2c6374958c4 kind: Event lastTimestamp: "2026-05-26T16:14:45Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-678784f46b-gshlx' metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:45Z" name: isvc-sklearn-graph-1-predictor-678784f46b.18b329a98ea9497d namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: d8ede924-8c14-4c32-b9ca-c714a121c1d2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: c539c745-98fe-4f92-a9e9-2a9c105a366f kind: Event lastTimestamp: "2026-05-26T16:14:45Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-678784f46b from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:45Z" name: isvc-sklearn-graph-1-predictor.18b329a98b6ea246 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 1e571028-4886-4e76-ab37-c168b0c5168d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:15:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 1c50d958-564f-43a2-8aa0-6239b84ae57e kind: Event lastTimestamp: "2026-05-26T16:15:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:45Z" name: isvc-sklearn-graph-1-predictor.18b329ad0a304bdb namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 94aa587c-8c0f-4525-91b1-a5e028b73223 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:15:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 1c50d958-564f-43a2-8aa0-6239b84ae57e kind: Event lastTimestamp: "2026-05-26T16:15:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:45Z" name: isvc-sklearn-graph-1-predictor.18b329ad0a99f849 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 24d31d81-1dba-44f0-a780-eaa2b631ad52 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:16:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 1c50d958-564f-43a2-8aa0-6239b84ae57e kind: Event lastTimestamp: "2026-05-26T16:16:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:31Z" name: isvc-sklearn-graph-1-predictor.18b329bb058627b5 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 564cae5f-3a80-4f0a-b62a-97b885dbf700 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:16:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 1c50d958-564f-43a2-8aa0-6239b84ae57e kind: Event lastTimestamp: "2026-05-26T16:16:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:31Z" name: isvc-sklearn-graph-1-predictor.18b329bb05f2bf27 namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 00c00728-c7b6-4722-bfcf-a72bea3b2af1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:14:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: f05a6ff7-e834-4e59-a920-ea2c74d2c520 kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:17Z" name: isvc-sklearn-graph-1.18b329a98ba62a04 namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 0f2c5302-6983-43cc-9448-882fa5c17c7e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: f05a6ff7-e834-4e59-a920-ea2c74d2c520 kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-05-26T16:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:17Z" name: isvc-sklearn-graph-1.18b329beebb0ad8c namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 6f93298d-8a71-476b-a256-de551cdd06d9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:14:47.031590Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:14:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:14:47Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329a9ce7703e6 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 629604b3-20ac-428a-a6cd-6ede02e4f808 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:14:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:47Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329a9d5c2bd57 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: 9c399143-6192-4a5e-8ece-38ee2a2b709c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:14:48Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:14:48Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329aa297c4a9e namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: f4c3ab74-5425-4065-8717-8d127882878f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:14:48Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" metadata: creationTimestamp: "2026-05-26T16:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:48Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329aa2b0223a4 namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: d2ceb53e-1dce-4948-84bf-6c5f665813b6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:14:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" in 1.483s (1.483s including waiting). Image size: 301466548 bytes.' metadata: creationTimestamp: "2026-05-26T16:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:50Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329aa837114e2 namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: 6e53a59b-f99c-4636-b5d3-ddee2c8dd617 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:14:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:50Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329aa8747b53f namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 3ef98cb4-8771-4fa9-9c5d-84aaa5e726db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:14:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:50Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329aa87d0eb87 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: 0ed74c30-8f5a-4a1a-8260-0b45dbb9225c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:14:54Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T16:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:54Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329ab8f0e586f namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: eb8104d5-a0d9-4b8c-aea1-2e79d85c938c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:15:08Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 14.379s (14.379s including waiting). Image size: 1560933300 bytes.' metadata: creationTimestamp: "2026-05-26T16:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:08Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329aee81e95f1 namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: 5d0c1ec1-0356-48f4-b7d3-397391e5c3ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:15:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:15:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:09Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329aeeba58240 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: 752ef8ea-87a9-4ebc-9b5f-d0585342e4f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:15:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:15:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:09Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329aeec120614 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: f2f1abdc-c328-465c-8cfa-e641246acc03 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:15:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T16:15:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:09Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329aeec1f6aff namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: e4c4db20-1623-457a-ba00-84d446542d11 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:15:11Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.524s (2.524s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:11Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329af82923d2f namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: d8a20ed4-0ead-4d9b-bab1-80bf34f8bab1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:15:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:11Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329af863841dc namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 5d3149cc-aa95-447d-8649-715ec71dfcde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:15:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:11Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329af86dce21e namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: d1f2492e-3620-4485-8068-cb759ac01f32 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:15:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:16:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:58Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329afc499951f namespace: kserve-ci-e2e-test resourceVersion: "16738" uid: 4d8dfaa1-e9af-4bef-8922-da10830f495c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:56Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329c7d60bbc48 namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: 3e3b866c-eaca-4d47-a47e-ba18ab1bc260 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:56Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329c7d60f4699 namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: 5145e464-ef10-4bc9-9ae0-88dcdb4ef77b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: d528a173-7324-4e32-9844-a453d6c16eb2 kind: Event lastTimestamp: "2026-05-26T16:16:58Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:58Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc.18b329c872cb9199 namespace: kserve-ci-e2e-test resourceVersion: "16736" uid: 02f3e9b1-05d0-4ae2-a3c0-f2f88132ebba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-54c99bfc7d namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: bcda9242-c7a5-4bf4-9e6c-2e363e22a488 kind: Event lastTimestamp: "2026-05-26T16:14:47Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-54c99bfc7d-n7tbc' metadata: creationTimestamp: "2026-05-26T16:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:47Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d.18b329a9cd9e7b85 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: c2a917da-cb64-4a4f-a7f6-3c88198d752b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: ccba09ed-fa92-499a-afea-f3a6d5efc386 kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-54c99bfc7d from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:46Z" name: isvc-sklearn-graph-2-predictor.18b329a9b6d7d6d3 namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 0cfb4950-dee8-4e21-925e-0e05d5a76fcd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:15:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 190070b3-dc8d-4d07-8e6b-fe87c8884425 kind: Event lastTimestamp: "2026-05-26T16:15:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:31Z" name: isvc-sklearn-graph-2-predictor.18b329ad3555b58a namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: 5b771b49-3d96-44f3-beac-c3e828c3c573 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:15:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 190070b3-dc8d-4d07-8e6b-fe87c8884425 kind: Event lastTimestamp: "2026-05-26T16:15:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:31Z" name: isvc-sklearn-graph-2-predictor.18b329ad361a196e namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: 34350109-b90a-4f9c-8928-5d7b5c354e46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 190070b3-dc8d-4d07-8e6b-fe87c8884425 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:46Z" name: isvc-sklearn-graph-2-predictor.18b329b7b258c578 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 06e5bfe2-1d15-4b51-9135-0b26dad8ddb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 190070b3-dc8d-4d07-8e6b-fe87c8884425 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:46Z" name: isvc-sklearn-graph-2-predictor.18b329b7b2c1ed52 namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 28c79832-c087-42b9-9ca8-df2ebbc3b4de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: 6bcd5881-6c63-4cf7-adbb-24c23da3d0a3 kind: Event lastTimestamp: "2026-05-26T16:16:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:18Z" name: isvc-sklearn-graph-2.18b329a9b6b53e6a namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: fe0dcd2f-8899-4066-8776-f0fadcc50280 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 6bcd5881-6c63-4cf7-adbb-24c23da3d0a3 kind: Event lastTimestamp: "2026-05-26T16:16:18Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-05-26T16:16:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:18Z" name: isvc-sklearn-graph-2.18b329bf27bec82f namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 26161d44-b4b8-4846-b214-caf40b8782a2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:14:46.418262Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-dk8qj to ip-10-0-131-126.ec2.internal metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:14:46Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329a9a9e86d9f namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: b8147e0e-0b5a-4878-8790-8c655e59384f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:46Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329a9b12aedd8 namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 7465872d-26d8-4298-b211-13da75c57e8a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:14:47Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:14:47Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329a9e70e1704 namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 9bf071a3-f786-4999-8529-b76dd2c51b54 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:14:47Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" metadata: creationTimestamp: "2026-05-26T16:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:47Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329a9e8dc4cc4 namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: e937d416-61b1-489c-9565-d2bbc5f75643 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:14:51Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0413900e48a4ba69f75c45b06af88f792921f4669631dcf3a047c1fe5c1fc341" in 4.088s (4.088s including waiting). Image size: 301466548 bytes.' metadata: creationTimestamp: "2026-05-26T16:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:51Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329aadc906f06 namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: c0bad063-5c3a-4381-a039-777b96179699 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:14:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T16:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:51Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329aae12d301f namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 5e92b5a7-50c3-4e70-a1b7-6d255134e96e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:14:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T16:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:51Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329aae1b0a145 namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: dc3f249b-1d30-4134-91c7-674f3c797bcc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:14:57Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-26T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:57Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329ac317cbf22 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: 4c797579-b961-4611-a6ef-255cf8d7b5d6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:15:22Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 25.205s (25.205s including waiting). Image size: 1306238203 bytes.' metadata: creationTimestamp: "2026-05-26T16:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:22Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329b20fe0c66e namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 4a29eaa2-aa9f-453e-8e9f-1aa7d271efc8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:15:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:22Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329b214204fe1 namespace: kserve-ci-e2e-test resourceVersion: "15524" uid: 6480b25e-fd08-43b1-a7cd-89dd670d105c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:15:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:22Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329b214951ffa namespace: kserve-ci-e2e-test resourceVersion: "15525" uid: 5ea4177a-4d1e-4713-8e3f-d0e6629a9eb9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:15:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:22Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329b214a12e60 namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 3ad8b279-7bfb-41cf-a898-22dde2e9a4ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:15:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:22Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329b217f53960 namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 70c2372d-53fa-469d-b7f1-d438c5524e2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:15:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:22Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329b218758f7f namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 12fcb084-7b85-4b4c-8bbe-dea330932851 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-26T16:15:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:16:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:15:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:20Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329b2803e928c namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 79235552-bce7-4e67-ac35-bd56fb7ca91f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329c7def19841 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 9c18ff18-0661-4acd-9eb1-5a0bf9dd70d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f9476710-d812-4e7d-ae40-bebf65649165 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-dk8qj.18b329c7def1e291 namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: 655223e1-60f3-48e6-b143-d35f0882b46f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-126.ec2.internal source: component: kubelet host: ip-10-0-131-126.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: 8b04d13d-1322-4f61-9481-ad3e059b6eff kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-dk8qj' metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:46Z" name: isvc-xgboost-graph-predictor-669d8d6456.18b329a9a97679ec namespace: kserve-ci-e2e-test resourceVersion: "15087" uid: 245d98c5-5637-4f99-bdb5-f4d3fd203902 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: ec2ec38a-94c6-4566-8175-12d1aa3d277e kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:46Z" name: isvc-xgboost-graph-predictor.18b329a99798446c namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: d23b7b9f-76fd-45e6-af39-0f830d0b960b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:15:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 4e9cdcf3-a815-4951-bf61-e191e3d9ec42 kind: Event lastTimestamp: "2026-05-26T16:15:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:31Z" name: isvc-xgboost-graph-predictor.18b329ad160ac65b namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 9abee0f4-4658-4f99-a760-a038bed0ef6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:15:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 4e9cdcf3-a815-4951-bf61-e191e3d9ec42 kind: Event lastTimestamp: "2026-05-26T16:15:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:31Z" name: isvc-xgboost-graph-predictor.18b329ad168aa08d namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 6917488f-6ef6-46a3-b0f5-a108c0ca79cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 4e9cdcf3-a815-4951-bf61-e191e3d9ec42 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:46Z" name: isvc-xgboost-graph-predictor.18b329b79285deef namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 119b1198-ccba-4188-aaa2-120e19c4bcc7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 4e9cdcf3-a815-4951-bf61-e191e3d9ec42 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:46Z" name: isvc-xgboost-graph-predictor.18b329b792eb2d86 namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: 943125a9-890d-4551-904b-d344fbb326a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 3230ae83-fb49-4680-939f-94242e250487 kind: Event lastTimestamp: "2026-05-26T16:16:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:30Z" name: isvc-xgboost-graph.18b329a997724283 namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 432702b9-aaa2-4c7e-9889-665e568cd637 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: 3230ae83-fb49-4680-939f-94242e250487 kind: Event lastTimestamp: "2026-05-26T16:16:30Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-05-26T16:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:30Z" name: isvc-xgboost-graph.18b329c1e3446475 namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: db9d92b1-5662-4e0c-8b5d-ce9d5a9d61af reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:16:45.806153Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5cbf9648f-m44jf namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: a75381a8-944d-417c-83d5-82de0c255202 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5cbf9648f-m44jf to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:16:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:16:45Z" name: model-chainer-5cbf9648f-m44jf.18b329c575fb2521 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: f5c1a58a-0759-4a4e-a386-0ca9a06f1376 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:45Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5cbf9648f-m44jf namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: a75381a8-944d-417c-83d5-82de0c255202 kind: Event lastTimestamp: "2026-05-26T16:16:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:45Z" name: model-chainer-5cbf9648f-m44jf.18b329c57f38f49e namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: 4b0c4cc9-aed8-419d-80a3-fa6319078079 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:46Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5cbf9648f-m44jf namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: a75381a8-944d-417c-83d5-82de0c255202 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:16:46Z" name: model-chainer-5cbf9648f-m44jf.18b329c5b316b06a namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 3d405c5e-7f2d-4007-999f-635ade8d4b33 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cbf9648f-m44jf namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: a75381a8-944d-417c-83d5-82de0c255202 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:46Z" name: model-chainer-5cbf9648f-m44jf.18b329c5b44c0d22 namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: cdc5f0da-8bab-4a54-b865-3f2cb28296d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cbf9648f-m44jf namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: a75381a8-944d-417c-83d5-82de0c255202 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-05-26T16:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:46Z" name: model-chainer-5cbf9648f-m44jf.18b329c5b75157a3 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: c2f248cb-03e4-4558-82bc-1e86e67d568c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cbf9648f-m44jf namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: a75381a8-944d-417c-83d5-82de0c255202 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: Started container model-chainer metadata: creationTimestamp: "2026-05-26T16:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:46Z" name: model-chainer-5cbf9648f-m44jf.18b329c5b7bd86f0 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: 66f831e5-08c4-4e1c-b677-dc4d5102623c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cbf9648f-m44jf namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: a75381a8-944d-417c-83d5-82de0c255202 kind: Event lastTimestamp: "2026-05-26T16:16:55Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-05-26T16:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:55Z" name: model-chainer-5cbf9648f-m44jf.18b329c7d06e37a2 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 72e00e94-2bb9-4db7-9353-1462fe8bf884 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:16:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cbf9648f-m44jf namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: a75381a8-944d-417c-83d5-82de0c255202 kind: Event lastTimestamp: "2026-05-26T16:17:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:23Z" name: model-chainer-5cbf9648f-m44jf.18b329c8623f55f9 namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: b09324cb-b060-4a5f-a3f8-96d8cbdc604e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5cbf9648f namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: 09c9395e-f75e-4ca0-b847-d89b4f0fe9b1 kind: Event lastTimestamp: "2026-05-26T16:16:45Z" message: 'Created pod: model-chainer-5cbf9648f-m44jf' metadata: creationTimestamp: "2026-05-26T16:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:45Z" name: model-chainer-5cbf9648f.18b329c5756e6fef namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: cfc80da5-4cb4-406b-8137-309e44caa679 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: a0ea9c42-ae55-4ef1-956b-0ddc8504416d kind: Event lastTimestamp: "2026-05-26T16:16:45Z" message: Scaled up replica set model-chainer-5cbf9648f from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:45Z" name: model-chainer.18b329c574b526d0 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: b9ea455d-36b0-4851-8f4f-c6ffb99be6cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: ed8460de-5911-43c0-b1c5-85ccedca1849 kind: Event lastTimestamp: "2026-05-26T16:16:45Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-05-26T16:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:45Z" name: model-chainer.18b329c577fcf215 namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: 5f790f42-fb26-4dac-9383-72fdeec36542 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:28:21.246131Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-86373-6d887b9df8-c9zkb namespace: kserve-ci-e2e-test resourceVersion: "22542" uid: a1df1e87-7cd1-4f46-8782-ba4dd1bd73ac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-86373-6d887b9df8-c9zkb to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:28:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:28:21Z" name: sequence-graph-86373-6d887b9df8-c9zkb.18b32a67616ef526 namespace: kserve-ci-e2e-test resourceVersion: "22548" uid: 0dbc030e-7c43-4092-9dba-16b6becd0169 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:21Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-86373-6d887b9df8-c9zkb namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: a1df1e87-7cd1-4f46-8782-ba4dd1bd73ac kind: Event lastTimestamp: "2026-05-26T16:28:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-86373-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:28:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:28:21Z" name: sequence-graph-86373-6d887b9df8-c9zkb.18b32a67684e0548 namespace: kserve-ci-e2e-test resourceVersion: "22562" uid: 8eba533c-acc0-41df-a38b-325aa9258d7d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:22Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-86373-6d887b9df8-c9zkb namespace: kserve-ci-e2e-test resourceVersion: "22551" uid: a1df1e87-7cd1-4f46-8782-ba4dd1bd73ac kind: Event lastTimestamp: "2026-05-26T16:28:22Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:28:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:28:22Z" name: sequence-graph-86373-6d887b9df8-c9zkb.18b32a679e5a5787 namespace: kserve-ci-e2e-test resourceVersion: "22573" uid: 7a6a033b-1893-4b3b-b5bd-86f307053640 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-86373} kind: Pod name: sequence-graph-86373-6d887b9df8-c9zkb namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: a1df1e87-7cd1-4f46-8782-ba4dd1bd73ac kind: Event lastTimestamp: "2026-05-26T16:28:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:28:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:28:22Z" name: sequence-graph-86373-6d887b9df8-c9zkb.18b32a679fa100d4 namespace: kserve-ci-e2e-test resourceVersion: "22575" uid: 4c18854b-d74c-4c5f-9040-f71cc541ff0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-86373} kind: Pod name: sequence-graph-86373-6d887b9df8-c9zkb namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: a1df1e87-7cd1-4f46-8782-ba4dd1bd73ac kind: Event lastTimestamp: "2026-05-26T16:28:22Z" message: 'Created container: sequence-graph-86373' metadata: creationTimestamp: "2026-05-26T16:28:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:28:22Z" name: sequence-graph-86373-6d887b9df8-c9zkb.18b32a67a365d922 namespace: kserve-ci-e2e-test resourceVersion: "22576" uid: 8cd873de-4bba-43e7-968c-6ca6ecf8348d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-86373} kind: Pod name: sequence-graph-86373-6d887b9df8-c9zkb namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: a1df1e87-7cd1-4f46-8782-ba4dd1bd73ac kind: Event lastTimestamp: "2026-05-26T16:28:22Z" message: Started container sequence-graph-86373 metadata: creationTimestamp: "2026-05-26T16:28:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:28:22Z" name: sequence-graph-86373-6d887b9df8-c9zkb.18b32a67a3d81010 namespace: kserve-ci-e2e-test resourceVersion: "22577" uid: b4479f8b-f5ce-4d30-a4ff-97c93072900b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-86373} kind: Pod name: sequence-graph-86373-6d887b9df8-c9zkb namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: a1df1e87-7cd1-4f46-8782-ba4dd1bd73ac kind: Event lastTimestamp: "2026-05-26T16:36:35Z" message: Stopping container sequence-graph-86373 metadata: creationTimestamp: "2026-05-26T16:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:35Z" name: sequence-graph-86373-6d887b9df8-c9zkb.18b32ada8e0bcb65 namespace: kserve-ci-e2e-test resourceVersion: "26431" uid: 3ea2f1e9-78fe-42be-b82a-18cfa7304e7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:36:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-86373} kind: Pod name: sequence-graph-86373-6d887b9df8-c9zkb namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: a1df1e87-7cd1-4f46-8782-ba4dd1bd73ac kind: Event lastTimestamp: "2026-05-26T16:37:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:03Z" name: sequence-graph-86373-6d887b9df8-c9zkb.18b32adb44196cc7 namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 3a03bc10-cdcb-42de-b3ea-8275b930823d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-86373-6d887b9df8 namespace: kserve-ci-e2e-test resourceVersion: "22538" uid: 8c277799-8b6a-4d32-8362-479cec877440 kind: Event lastTimestamp: "2026-05-26T16:28:21Z" message: 'Created pod: sequence-graph-86373-6d887b9df8-c9zkb' metadata: creationTimestamp: "2026-05-26T16:28:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:21Z" name: sequence-graph-86373-6d887b9df8.18b32a6760de1f19 namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 417d27b7-e287-41c5-bbfd-7a44c3a03eab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-86373 namespace: kserve-ci-e2e-test resourceVersion: "22535" uid: 44117dcd-a16d-4453-9633-0a786b7cbac4 kind: Event lastTimestamp: "2026-05-26T16:28:21Z" message: Scaled up replica set sequence-graph-86373-6d887b9df8 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:28:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:21Z" name: sequence-graph-86373.18b32a67601a7d4f namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: b912b54e-0025-4d36-ae12-d228ee9451bb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-86373 namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: e0e1faa0-613c-4620-bbb9-35080edf11a2 kind: Event lastTimestamp: "2026-05-26T16:28:21Z" message: InferenceGraph [sequence-graph-86373] is Ready metadata: creationTimestamp: "2026-05-26T16:28:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:28:21Z" name: sequence-graph-86373.18b32a6763f1abd0 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 1537b58a-3686-42aa-8bde-bf4f7d66ecf9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-86373 namespace: kserve-ci-e2e-test resourceVersion: "22537" uid: 212d30d4-715e-4641-8897-559a7c9141d7 kind: Event lastTimestamp: "2026-05-26T16:28:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:28:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:36Z" name: sequence-graph-86373.18b32a6ade6da06f namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: 6f476d6a-9f98-4900-a1c2-a3c3ffd698ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-86373 namespace: kserve-ci-e2e-test resourceVersion: "22537" uid: 212d30d4-715e-4641-8897-559a7c9141d7 kind: Event lastTimestamp: "2026-05-26T16:28:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:28:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:36Z" name: sequence-graph-86373.18b32a6adea77a29 namespace: kserve-ci-e2e-test resourceVersion: "22680" uid: ed7cf205-53e7-4b11-b973-8715419c132a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-86373 namespace: kserve-ci-e2e-test resourceVersion: "22677" uid: 212d30d4-715e-4641-8897-559a7c9141d7 kind: Event lastTimestamp: "2026-05-26T16:28:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:28:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:51Z" name: sequence-graph-86373.18b32a6e5d547699 namespace: kserve-ci-e2e-test resourceVersion: "22785" uid: ce273766-bcb9-48a5-9352-7c66131f2e3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-86373 namespace: kserve-ci-e2e-test resourceVersion: "22677" uid: 212d30d4-715e-4641-8897-559a7c9141d7 kind: Event lastTimestamp: "2026-05-26T16:28:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:28:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:51Z" name: sequence-graph-86373.18b32a6e5dba4fd3 namespace: kserve-ci-e2e-test resourceVersion: "22787" uid: 50f31b5f-28a8-474b-98c2-5105ed2770db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:27:10.957917Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a007a-5bd657f6bf-98m9p namespace: kserve-ci-e2e-test resourceVersion: "21717" uid: a5e0edc4-b648-4966-8d61-5e288e8ec0b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a007a-5bd657f6bf-98m9p to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:27:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:27:10Z" name: sequence-graph-a007a-5bd657f6bf-98m9p.18b32a5703edefcc namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 3b3504e7-3524-4159-9be2-8243da6410b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a007a-5bd657f6bf-98m9p namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: a5e0edc4-b648-4966-8d61-5e288e8ec0b8 kind: Event lastTimestamp: "2026-05-26T16:27:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-a007a-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:11Z" name: sequence-graph-a007a-5bd657f6bf-98m9p.18b32a57122e47c6 namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 2a2214f2-9f85-4a2c-bf26-1bacf2540a4d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a007a-5bd657f6bf-98m9p namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: a5e0edc4-b648-4966-8d61-5e288e8ec0b8 kind: Event lastTimestamp: "2026-05-26T16:27:11Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:27:11Z" name: sequence-graph-a007a-5bd657f6bf-98m9p.18b32a574129eb37 namespace: kserve-ci-e2e-test resourceVersion: "21755" uid: fa09cc6e-5bcd-4275-8d6a-505c75f8aa8a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a007a} kind: Pod name: sequence-graph-a007a-5bd657f6bf-98m9p namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: a5e0edc4-b648-4966-8d61-5e288e8ec0b8 kind: Event lastTimestamp: "2026-05-26T16:27:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:12Z" name: sequence-graph-a007a-5bd657f6bf-98m9p.18b32a574266e7db namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: a3761f4a-ac03-41bf-a657-89b4727b69c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a007a} kind: Pod name: sequence-graph-a007a-5bd657f6bf-98m9p namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: a5e0edc4-b648-4966-8d61-5e288e8ec0b8 kind: Event lastTimestamp: "2026-05-26T16:27:12Z" message: 'Created container: sequence-graph-a007a' metadata: creationTimestamp: "2026-05-26T16:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:12Z" name: sequence-graph-a007a-5bd657f6bf-98m9p.18b32a5745536137 namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: c6f61147-ff66-4dcb-92b2-7cddc195fa75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a007a} kind: Pod name: sequence-graph-a007a-5bd657f6bf-98m9p namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: a5e0edc4-b648-4966-8d61-5e288e8ec0b8 kind: Event lastTimestamp: "2026-05-26T16:27:12Z" message: Started container sequence-graph-a007a metadata: creationTimestamp: "2026-05-26T16:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:12Z" name: sequence-graph-a007a-5bd657f6bf-98m9p.18b32a5745cef2f7 namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: aa3725bc-0b97-455e-ad25-05aa99376b20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a007a} kind: Pod name: sequence-graph-a007a-5bd657f6bf-98m9p namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: a5e0edc4-b648-4966-8d61-5e288e8ec0b8 kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: Stopping container sequence-graph-a007a metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:21Z" name: sequence-graph-a007a-5bd657f6bf-98m9p.18b32a595c48811f namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: 7beb894f-765a-471b-ba33-adf9e16e67f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:27:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a007a} kind: Pod name: sequence-graph-a007a-5bd657f6bf-98m9p namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: a5e0edc4-b648-4966-8d61-5e288e8ec0b8 kind: Event lastTimestamp: "2026-05-26T16:27:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:27:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:48Z" name: sequence-graph-a007a-5bd657f6bf-98m9p.18b32a59feffe4b9 namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: e5f4b459-8c34-43ef-b17c-8884d8c0ba51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a007a-5bd657f6bf namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: 9dcedb9a-b2ab-4491-999f-a37773e27b0a kind: Event lastTimestamp: "2026-05-26T16:27:10Z" message: 'Created pod: sequence-graph-a007a-5bd657f6bf-98m9p' metadata: creationTimestamp: "2026-05-26T16:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:10Z" name: sequence-graph-a007a-5bd657f6bf.18b32a57038073d3 namespace: kserve-ci-e2e-test resourceVersion: "21718" uid: be2edbd8-65de-44f6-8157-13fb1b486b8e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a007a namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: be03c8e5-a12e-4558-945d-517ad0014c3e kind: Event lastTimestamp: "2026-05-26T16:27:10Z" message: Scaled up replica set sequence-graph-a007a-5bd657f6bf from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:10Z" name: sequence-graph-a007a.18b32a5702e40d09 namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: 6906267f-a4cd-466e-8615-e687c62c2cc3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a007a namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 750a4102-ecf9-4ac5-81df-c985884e9093 kind: Event lastTimestamp: "2026-05-26T16:27:10Z" message: InferenceGraph [sequence-graph-a007a] is Ready metadata: creationTimestamp: "2026-05-26T16:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:11Z" name: sequence-graph-a007a.18b32a57066375c2 namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: 4c65d0a8-a74c-47b0-8885-d3e3bcc8e68b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:17:56.079783Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ad62d-6db94b859c-qz87l namespace: kserve-ci-e2e-test resourceVersion: "17286" uid: 11101453-8b37-40af-bdbc-76db842bb667 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-ad62d-6db94b859c-qz87l to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:17:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:17:56Z" name: sequence-graph-ad62d-6db94b859c-qz87l.18b329d5d29d86f9 namespace: kserve-ci-e2e-test resourceVersion: "17292" uid: 2d29b54a-4aa2-4580-9420-97da1f44fbe9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ad62d-6db94b859c-qz87l namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 11101453-8b37-40af-bdbc-76db842bb667 kind: Event lastTimestamp: "2026-05-26T16:17:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-ad62d-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:56Z" name: sequence-graph-ad62d-6db94b859c-qz87l.18b329d5dbb7c32c namespace: kserve-ci-e2e-test resourceVersion: "17306" uid: cf22a15c-1f1b-4f0d-8122-7dc2afb14136 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ad62d-6db94b859c-qz87l namespace: kserve-ci-e2e-test resourceVersion: "17297" uid: 11101453-8b37-40af-bdbc-76db842bb667 kind: Event lastTimestamp: "2026-05-26T16:17:57Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:17:57Z" name: sequence-graph-ad62d-6db94b859c-qz87l.18b329d60f7499c8 namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 908322da-d993-469e-88f5-56b1f255f129 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ad62d} kind: Pod name: sequence-graph-ad62d-6db94b859c-qz87l namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 11101453-8b37-40af-bdbc-76db842bb667 kind: Event lastTimestamp: "2026-05-26T16:17:57Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" metadata: creationTimestamp: "2026-05-26T16:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:57Z" name: sequence-graph-ad62d-6db94b859c-qz87l.18b329d610affe6b namespace: kserve-ci-e2e-test resourceVersion: "17316" uid: f83baaea-0ffe-4045-8831-c0a724510e19 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ad62d} kind: Pod name: sequence-graph-ad62d-6db94b859c-qz87l namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 11101453-8b37-40af-bdbc-76db842bb667 kind: Event lastTimestamp: "2026-05-26T16:17:58Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" in 1.858s (1.858s including waiting). Image size: 217997401 bytes.' metadata: creationTimestamp: "2026-05-26T16:17:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:58Z" name: sequence-graph-ad62d-6db94b859c-qz87l.18b329d67f782c2b namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 8704ca58-04b9-4330-ab8f-e852cb2afb6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ad62d} kind: Pod name: sequence-graph-ad62d-6db94b859c-qz87l namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 11101453-8b37-40af-bdbc-76db842bb667 kind: Event lastTimestamp: "2026-05-26T16:17:59Z" message: 'Created container: sequence-graph-ad62d' metadata: creationTimestamp: "2026-05-26T16:17:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:59Z" name: sequence-graph-ad62d-6db94b859c-qz87l.18b329d682b23886 namespace: kserve-ci-e2e-test resourceVersion: "17333" uid: 2480a7f4-0aa8-40cc-8fce-5526fc25a3c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ad62d} kind: Pod name: sequence-graph-ad62d-6db94b859c-qz87l namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 11101453-8b37-40af-bdbc-76db842bb667 kind: Event lastTimestamp: "2026-05-26T16:17:59Z" message: Started container sequence-graph-ad62d metadata: creationTimestamp: "2026-05-26T16:17:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:59Z" name: sequence-graph-ad62d-6db94b859c-qz87l.18b329d683647433 namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 636e56f7-a35f-4fd9-8279-82923c0dd29b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ad62d} kind: Pod name: sequence-graph-ad62d-6db94b859c-qz87l namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 11101453-8b37-40af-bdbc-76db842bb667 kind: Event lastTimestamp: "2026-05-26T16:26:10Z" message: Stopping container sequence-graph-ad62d metadata: creationTimestamp: "2026-05-26T16:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:10Z" name: sequence-graph-ad62d-6db94b859c-qz87l.18b32a4903c5ca48 namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 4c2b790d-6ab3-429b-88b0-7af658c4bf06 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:26:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ad62d} kind: Pod name: sequence-graph-ad62d-6db94b859c-qz87l namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 11101453-8b37-40af-bdbc-76db842bb667 kind: Event lastTimestamp: "2026-05-26T16:26:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:40Z" name: sequence-graph-ad62d-6db94b859c-qz87l.18b32a4a05af427e namespace: kserve-ci-e2e-test resourceVersion: "21294" uid: a918f767-a617-4f5f-9bed-53378dd8b4b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-ad62d-6db94b859c namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 0d103a03-dcfe-48a8-b0f8-26a9618c9c1c kind: Event lastTimestamp: "2026-05-26T16:17:56Z" message: 'Created pod: sequence-graph-ad62d-6db94b859c-qz87l' metadata: creationTimestamp: "2026-05-26T16:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:56Z" name: sequence-graph-ad62d-6db94b859c.18b329d5d22a8e44 namespace: kserve-ci-e2e-test resourceVersion: "17288" uid: 2b53da17-77c1-471d-952d-e6e9b0459d5a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-ad62d namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 97c76945-b44e-4d6f-9448-f3352a6432fa kind: Event lastTimestamp: "2026-05-26T16:17:56Z" message: Scaled up replica set sequence-graph-ad62d-6db94b859c from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:56Z" name: sequence-graph-ad62d.18b329d5d1346ed3 namespace: kserve-ci-e2e-test resourceVersion: "17282" uid: c92b8d46-a3ed-4670-af0d-93dbb27a92a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ad62d namespace: kserve-ci-e2e-test resourceVersion: "17299" uid: 58be40f1-730e-4c15-afe5-a3d5bc5607aa kind: Event lastTimestamp: "2026-05-26T16:17:56Z" message: InferenceGraph [sequence-graph-ad62d] is Ready metadata: creationTimestamp: "2026-05-26T16:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:56Z" name: sequence-graph-ad62d.18b329d5d4abc019 namespace: kserve-ci-e2e-test resourceVersion: "17301" uid: 8c5e11fb-aa0c-4274-b2d7-ebd4f23d848a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:18:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ad62d namespace: kserve-ci-e2e-test resourceVersion: "17280" uid: 5ef2b55f-f667-4103-8f5c-7e895a87663b kind: Event lastTimestamp: "2026-05-26T16:18:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:41Z" name: sequence-graph-ad62d.18b329d94fa74549 namespace: kserve-ci-e2e-test resourceVersion: "17637" uid: 1d0a5a3e-f98c-4f51-92e1-0cd827767d2c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:18:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ad62d namespace: kserve-ci-e2e-test resourceVersion: "17280" uid: 5ef2b55f-f667-4103-8f5c-7e895a87663b kind: Event lastTimestamp: "2026-05-26T16:18:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:41Z" name: sequence-graph-ad62d.18b329d950092fe2 namespace: kserve-ci-e2e-test resourceVersion: "17638" uid: 165eff79-2a98-4c22-8806-118cde76a34a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:38:06.667420Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5798a-56688b886c-wnnxz namespace: kserve-ci-e2e-test resourceVersion: "27572" uid: d632a147-bbd1-4829-8e87-16f3c80e5185 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-5798a-56688b886c-wnnxz to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:38:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:38:06Z" name: splitter-graph-5798a-56688b886c-wnnxz.18b32aefaf42e538 namespace: kserve-ci-e2e-test resourceVersion: "27578" uid: 899c6e63-6c31-4bd4-b516-6825453f1eac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:06Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5798a-56688b886c-wnnxz namespace: kserve-ci-e2e-test resourceVersion: "27574" uid: d632a147-bbd1-4829-8e87-16f3c80e5185 kind: Event lastTimestamp: "2026-05-26T16:38:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-5798a-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:38:06Z" name: splitter-graph-5798a-56688b886c-wnnxz.18b32aefbb063751 namespace: kserve-ci-e2e-test resourceVersion: "27594" uid: a6295ecb-7de6-49da-a707-34032baba25b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:07Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5798a-56688b886c-wnnxz namespace: kserve-ci-e2e-test resourceVersion: "27581" uid: d632a147-bbd1-4829-8e87-16f3c80e5185 kind: Event lastTimestamp: "2026-05-26T16:38:07Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:38:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:38:07Z" name: splitter-graph-5798a-56688b886c-wnnxz.18b32aefec1b910a namespace: kserve-ci-e2e-test resourceVersion: "27603" uid: d147db8c-2869-4bb9-a3dc-1309aad0294b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5798a} kind: Pod name: splitter-graph-5798a-56688b886c-wnnxz namespace: kserve-ci-e2e-test resourceVersion: "27574" uid: d632a147-bbd1-4829-8e87-16f3c80e5185 kind: Event lastTimestamp: "2026-05-26T16:38:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:38:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:38:07Z" name: splitter-graph-5798a-56688b886c-wnnxz.18b32aefede8bf36 namespace: kserve-ci-e2e-test resourceVersion: "27605" uid: 25534cb9-c021-48a0-a4b1-244c7b36985b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5798a} kind: Pod name: splitter-graph-5798a-56688b886c-wnnxz namespace: kserve-ci-e2e-test resourceVersion: "27574" uid: d632a147-bbd1-4829-8e87-16f3c80e5185 kind: Event lastTimestamp: "2026-05-26T16:38:07Z" message: 'Created container: splitter-graph-5798a' metadata: creationTimestamp: "2026-05-26T16:38:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:38:07Z" name: splitter-graph-5798a-56688b886c-wnnxz.18b32aeff16ac683 namespace: kserve-ci-e2e-test resourceVersion: "27606" uid: ad26eab0-63f4-4fa3-87bf-e91a75d3f770 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5798a} kind: Pod name: splitter-graph-5798a-56688b886c-wnnxz namespace: kserve-ci-e2e-test resourceVersion: "27574" uid: d632a147-bbd1-4829-8e87-16f3c80e5185 kind: Event lastTimestamp: "2026-05-26T16:38:07Z" message: Started container splitter-graph-5798a metadata: creationTimestamp: "2026-05-26T16:38:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:38:07Z" name: splitter-graph-5798a-56688b886c-wnnxz.18b32aeff1dadc38 namespace: kserve-ci-e2e-test resourceVersion: "27607" uid: db86b442-d6d0-4b64-8c3b-82c713821ad4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5798a} kind: Pod name: splitter-graph-5798a-56688b886c-wnnxz namespace: kserve-ci-e2e-test resourceVersion: "27574" uid: d632a147-bbd1-4829-8e87-16f3c80e5185 kind: Event lastTimestamp: "2026-05-26T16:46:21Z" message: Stopping container splitter-graph-5798a metadata: creationTimestamp: "2026-05-26T16:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:46:21Z" name: splitter-graph-5798a-56688b886c-wnnxz.18b32b62d91cbe41 namespace: kserve-ci-e2e-test resourceVersion: "31032" uid: 64cd493a-16bd-4b08-96b1-0a9da1288d34 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:46:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5798a} kind: Pod name: splitter-graph-5798a-56688b886c-wnnxz namespace: kserve-ci-e2e-test resourceVersion: "27574" uid: d632a147-bbd1-4829-8e87-16f3c80e5185 kind: Event lastTimestamp: "2026-05-26T16:46:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:46:49Z" name: splitter-graph-5798a-56688b886c-wnnxz.18b32b63a3bf9a0d namespace: kserve-ci-e2e-test resourceVersion: "31267" uid: 995f6b61-b9bb-4d8a-929f-833d20540462 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-5798a-56688b886c namespace: kserve-ci-e2e-test resourceVersion: "27569" uid: d0b06b2f-5f46-46e8-b276-6271c09316ca kind: Event lastTimestamp: "2026-05-26T16:38:06Z" message: 'Created pod: splitter-graph-5798a-56688b886c-wnnxz' metadata: creationTimestamp: "2026-05-26T16:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:06Z" name: splitter-graph-5798a-56688b886c.18b32aefaecbbcdf namespace: kserve-ci-e2e-test resourceVersion: "27573" uid: d88b685c-dcc0-4de5-bc5b-b481f9679255 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-5798a namespace: kserve-ci-e2e-test resourceVersion: "27567" uid: 834f6dc2-c5dc-47ad-9780-c4653d9adf9d kind: Event lastTimestamp: "2026-05-26T16:38:06Z" message: Scaled up replica set splitter-graph-5798a-56688b886c from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:06Z" name: splitter-graph-5798a.18b32aefadf0c839 namespace: kserve-ci-e2e-test resourceVersion: "27570" uid: 7c82a950-f33f-4b4f-9e2c-035bbdda179e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-5798a namespace: kserve-ci-e2e-test resourceVersion: "27583" uid: befe9660-7a0b-486f-b785-831e9ae65690 kind: Event lastTimestamp: "2026-05-26T16:38:06Z" message: InferenceGraph [splitter-graph-5798a] is Ready metadata: creationTimestamp: "2026-05-26T16:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:38:06Z" name: splitter-graph-5798a.18b32aefb19bbb5b namespace: kserve-ci-e2e-test resourceVersion: "27585" uid: 6eb818c4-788f-475b-bd95-71a531512b40 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-5798a namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: befe9660-7a0b-486f-b785-831e9ae65690 kind: Event lastTimestamp: "2026-05-26T16:38:06Z" message: 'Failed to update status for InferenceGraph "splitter-graph-5798a": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-5798a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:38:06Z" name: splitter-graph-5798a.18b32aefb49db9c3 namespace: kserve-ci-e2e-test resourceVersion: "27590" uid: 1c751fed-a4d2-40ee-b6ff-5efb769742b0 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-5798a namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: befe9660-7a0b-486f-b785-831e9ae65690 kind: Event lastTimestamp: "2026-05-26T16:38:06Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-5798a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:38:06Z" name: splitter-graph-5798a.18b32aefb49e1223 namespace: kserve-ci-e2e-test resourceVersion: "27591" uid: 2696996e-9327-4c7d-8907-845380c6c064 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5798a namespace: kserve-ci-e2e-test resourceVersion: "27568" uid: 343b0e24-616e-4714-9559-c20f8034e5e9 kind: Event lastTimestamp: "2026-05-26T16:38:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:21Z" name: splitter-graph-5798a.18b32af32c5a850a namespace: kserve-ci-e2e-test resourceVersion: "27717" uid: 8c0ab834-3c5d-4bad-b02c-d89b96d6701b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5798a namespace: kserve-ci-e2e-test resourceVersion: "27568" uid: 343b0e24-616e-4714-9559-c20f8034e5e9 kind: Event lastTimestamp: "2026-05-26T16:38:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:21Z" name: splitter-graph-5798a.18b32af32ccbd4b7 namespace: kserve-ci-e2e-test resourceVersion: "27719" uid: 201be530-119a-4487-b48f-a049d9d8cc6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5798a namespace: kserve-ci-e2e-test resourceVersion: "27718" uid: 343b0e24-616e-4714-9559-c20f8034e5e9 kind: Event lastTimestamp: "2026-05-26T16:38:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:36Z" name: splitter-graph-5798a.18b32af6ab5cc99c namespace: kserve-ci-e2e-test resourceVersion: "27820" uid: f78a4a4b-c716-4242-b663-ae1c6e23a90b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:38:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5798a namespace: kserve-ci-e2e-test resourceVersion: "27718" uid: 343b0e24-616e-4714-9559-c20f8034e5e9 kind: Event lastTimestamp: "2026-05-26T16:38:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:36Z" name: splitter-graph-5798a.18b32af6abbfb46d namespace: kserve-ci-e2e-test resourceVersion: "27822" uid: 6d0280cf-a38a-4731-b6ca-d89cd9f3f069 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:36:56.442039Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-82fbc-dbf78cc7b-2vw2x namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: 533c8237-ca5f-421b-8004-0c57d78ace35 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-82fbc-dbf78cc7b-2vw2x to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:36:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:36:56Z" name: splitter-graph-82fbc-dbf78cc7b-2vw2x.18b32adf5580a052 namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: 24494c87-3ba6-4ef3-bc3d-1e51c29a8ecb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:56Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-82fbc-dbf78cc7b-2vw2x namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 533c8237-ca5f-421b-8004-0c57d78ace35 kind: Event lastTimestamp: "2026-05-26T16:36:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-82fbc-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:56Z" name: splitter-graph-82fbc-dbf78cc7b-2vw2x.18b32adf5f6daf21 namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: 372101c6-127a-459d-b986-206489ac14ba reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:57Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-82fbc-dbf78cc7b-2vw2x namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 533c8237-ca5f-421b-8004-0c57d78ace35 kind: Event lastTimestamp: "2026-05-26T16:36:57Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:36:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:36:57Z" name: splitter-graph-82fbc-dbf78cc7b-2vw2x.18b32adf92571688 namespace: kserve-ci-e2e-test resourceVersion: "26785" uid: 22e76c07-96fa-42ff-be87-c240dc2ee8c7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-82fbc} kind: Pod name: splitter-graph-82fbc-dbf78cc7b-2vw2x namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 533c8237-ca5f-421b-8004-0c57d78ace35 kind: Event lastTimestamp: "2026-05-26T16:36:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:36:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:57Z" name: splitter-graph-82fbc-dbf78cc7b-2vw2x.18b32adf93c246e1 namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 48e778a5-5eed-4b9c-a897-80cd359f6908 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-82fbc} kind: Pod name: splitter-graph-82fbc-dbf78cc7b-2vw2x namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 533c8237-ca5f-421b-8004-0c57d78ace35 kind: Event lastTimestamp: "2026-05-26T16:36:57Z" message: 'Created container: splitter-graph-82fbc' metadata: creationTimestamp: "2026-05-26T16:36:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:57Z" name: splitter-graph-82fbc-dbf78cc7b-2vw2x.18b32adf96e51b3b namespace: kserve-ci-e2e-test resourceVersion: "26788" uid: 54e013f1-f75e-4534-898b-85664fe23f83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-82fbc} kind: Pod name: splitter-graph-82fbc-dbf78cc7b-2vw2x namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 533c8237-ca5f-421b-8004-0c57d78ace35 kind: Event lastTimestamp: "2026-05-26T16:36:57Z" message: Started container splitter-graph-82fbc metadata: creationTimestamp: "2026-05-26T16:36:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:57Z" name: splitter-graph-82fbc-dbf78cc7b-2vw2x.18b32adf975507c4 namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 467dbe85-bec8-441e-8086-549a317dcec0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-82fbc} kind: Pod name: splitter-graph-82fbc-dbf78cc7b-2vw2x namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 533c8237-ca5f-421b-8004-0c57d78ace35 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: Stopping container splitter-graph-82fbc metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:06Z" name: splitter-graph-82fbc-dbf78cc7b-2vw2x.18b32ae1aa56f7a0 namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 013f930d-53c5-4fae-9d76-7190cec0e519 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:37:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-82fbc} kind: Pod name: splitter-graph-82fbc-dbf78cc7b-2vw2x namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 533c8237-ca5f-421b-8004-0c57d78ace35 kind: Event lastTimestamp: "2026-05-26T16:37:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:34Z" name: splitter-graph-82fbc-dbf78cc7b-2vw2x.18b32ae25e24ad0b namespace: kserve-ci-e2e-test resourceVersion: "27247" uid: fb207512-ae72-4e28-a703-379c2e6ec4da reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-82fbc-dbf78cc7b namespace: kserve-ci-e2e-test resourceVersion: "26751" uid: 00d0511e-9ac2-4dd7-9345-6b3f72a0bbe7 kind: Event lastTimestamp: "2026-05-26T16:36:56Z" message: 'Created pod: splitter-graph-82fbc-dbf78cc7b-2vw2x' metadata: creationTimestamp: "2026-05-26T16:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:56Z" name: splitter-graph-82fbc-dbf78cc7b.18b32adf5470025e namespace: kserve-ci-e2e-test resourceVersion: "26756" uid: 97f1b7f8-ae31-4001-ab04-36abf2176d70 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-82fbc namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: 4c5293e4-bbd0-4f34-a584-051330f93035 kind: Event lastTimestamp: "2026-05-26T16:36:56Z" message: Scaled up replica set splitter-graph-82fbc-dbf78cc7b from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:56Z" name: splitter-graph-82fbc.18b32adf535a7e9f namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: ac96dbc1-c0c4-45a2-bb7e-37c517ee7240 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-82fbc namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: 29901595-d0d6-4e5d-bc34-88692ddf08f5 kind: Event lastTimestamp: "2026-05-26T16:36:56Z" message: InferenceGraph [splitter-graph-82fbc] is Ready metadata: creationTimestamp: "2026-05-26T16:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:36:56Z" name: splitter-graph-82fbc.18b32adf5a9919f3 namespace: kserve-ci-e2e-test resourceVersion: "26771" uid: ea97a206-67f3-4fde-851e-50fb95651395 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:25:31.182526Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:25:31Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a3fc8da4602 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 37ace2f2-38b9-421e-b4d1-721f341b002c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-007a5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a3fd0dd953a namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: fd3f3712-4399-4bf2-b94a-ac97c19047ee reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:25:32Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:25:32Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a40062093a6 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: f7566c5d-e6e4-4fd9-a265-b5fb7a0e4569 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:25:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:32Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a4007796055 namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: 659b1b5c-b051-4f95-8faf-3200ab99cdc9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:25:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:32Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a400af4eb3d namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: fd136625-e8c5-472b-bad0-4d14130148ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:25:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:32Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a400b6db2f4 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 25ad9ccd-0393-499b-a0e2-d0b56a19d363 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:25:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:32Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a400b7a6f7e namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 829de4fe-a9e5-4548-ade5-fdb8c6b6febf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:25:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:32Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a400f62ab19 namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 69d3dd9e-2a1a-4f01-8ab8-23af88dcb463 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:25:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:32Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a400fe9e205 namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 55a70337-3589-43a4-a04b-d8aadfd15914 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:25:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:26:09Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:09Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a403ed513d6 namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 4b925c25-7c0f-4a34-91e8-09b4dc72801d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:41Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a5020160905 namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: fd7c4150-4f26-4ad3-a05d-e86f304d99bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:41Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a50201691c2 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 8740353f-79cb-465a-8d83-a5fd4d86c25e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 579e746b-3525-4694-b44a-495beaf07f83 kind: Event lastTimestamp: "2026-05-26T16:26:44Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:44Z" name: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8.18b32a50c6f44951 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: 5368cdeb-4d36-4f35-be05-ae217fcf0589 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-007a5-predictor-6b94cd4f88 namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: d3825fc4-f524-4d12-8217-8546110dbd13 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: 'Created pod: success-200-isvc-007a5-predictor-6b94cd4f88-tngw8' metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:25:31Z" name: success-200-isvc-007a5-predictor-6b94cd4f88.18b32a3fc8248157 namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: a4119546-fcd2-492e-8d64-f5c4b13b5293 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: c52d14f5-acce-44fc-98a9-f7b046d84aef kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Scaled up replica set success-200-isvc-007a5-predictor-6b94cd4f88 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:25:31Z" name: success-200-isvc-007a5-predictor.18b32a3fc3cc943a namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 9d8c2ade-dc7e-4ad8-9b3c-ab2307c875cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 0cd68236-0cf6-4236-8e74-6753defc5cff kind: Event lastTimestamp: "2026-05-26T16:26:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:01Z" name: success-200-isvc-007a5-predictor.18b32a43423a0939 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: d83f1811-2900-4f47-97b3-96a4f0111be9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 0cd68236-0cf6-4236-8e74-6753defc5cff kind: Event lastTimestamp: "2026-05-26T16:26:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:01Z" name: success-200-isvc-007a5-predictor.18b32a43427e0df3 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 75dbb307-a170-48a9-877c-fff9583491fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 0cd68236-0cf6-4236-8e74-6753defc5cff kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:31Z" name: success-200-isvc-007a5-predictor.18b32a4a403a0bc4 namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: d4a63be8-c3b5-4b3e-bbca-74b88734bd25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:26:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-007a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 0cd68236-0cf6-4236-8e74-6753defc5cff kind: Event lastTimestamp: "2026-05-26T16:26:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:31Z" name: success-200-isvc-007a5-predictor.18b32a4a40a2c653 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 93fb0c15-194f-4055-b8fc-f694fe3057af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-007a5 namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: b4330135-4e7b-4622-9523-269387f25db7 kind: Event lastTimestamp: "2026-05-26T16:26:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:19Z" name: success-200-isvc-007a5.18b32a3fc3a214fa namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 96cd6930-977c-42a4-80dc-e5e42f67b199 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-007a5 namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: b4330135-4e7b-4622-9523-269387f25db7 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-007a5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-007a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:25:31Z" name: success-200-isvc-007a5.18b32a3fc52baef0 namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: c68287ff-613e-48a5-8999-5e261c74a68c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-007a5 namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: b4330135-4e7b-4622-9523-269387f25db7 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-007a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:25:31Z" name: success-200-isvc-007a5.18b32a3fc52bfcdc namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 96d79aea-a169-4169-ba0e-f1c2be6e05b9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-007a5 namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: b4330135-4e7b-4622-9523-269387f25db7 kind: Event lastTimestamp: "2026-05-26T16:26:19Z" message: InferenceService [success-200-isvc-007a5] is Ready metadata: creationTimestamp: "2026-05-26T16:26:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:19Z" name: success-200-isvc-007a5.18b32a4af9357400 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: 9ed4cf43-16e4-429a-b5a1-efdc7bc64a8d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:26:41.404336Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2138e-predictor-5d4757c55b-zwp2f to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:26:41Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a5022661943 namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: cb26801c-a2d9-4e42-aaa6-34302eee0f6b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2138e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:41Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a5030203c92 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: cc1aacb2-8178-4dbe-af98-59895b05845d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:26:42Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a505fe2c03f namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: a1bd4018-daa3-49f1-803b-9c635408bf63 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a5061334a10 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 58dd7b53-82e7-4113-805f-6583fc824612 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a506489ac24 namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: f9de2eab-10b4-4c7d-abe4-824980379a6c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a5064fefd32 namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: bb6c5433-1826-4978-a803-192ff6835cba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a50650af30a namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: e122f272-b50e-4a5c-b31b-b84f31b691b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a506914d6c4 namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 8192cbae-1886-417f-9767-3304f86f2005 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:26:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:42Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a50698b82c6 namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 2736d32c-bded-4310-b4fd-07d222f1a333 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:26:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:27:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:19Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32a509a686af6 namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: bd41619c-4370-4ee8-95a8-9fce1be3c7af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32ad152a3bff6 namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: ec188a2d-328d-47d6-a104-37074e6fae10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dcd5a011-9458-4571-8981-038c5a4f73b7 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f.18b32ad152a4322c namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: 68c20e37-b40e-4cf0-a977-19aa0e5cded0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2138e-predictor-5d4757c55b namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 2781ffaf-c7cf-4b07-a79c-fe74004be88f kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: 'Created pod: success-200-isvc-2138e-predictor-5d4757c55b-zwp2f' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:41Z" name: success-200-isvc-2138e-predictor-5d4757c55b.18b32a5021f9eca3 namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: 408ce7ad-0b9e-4dd3-a0ea-548fe6144fd6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 2ab6108c-68b2-49c3-bda0-c488706eb183 kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: Scaled up replica set success-200-isvc-2138e-predictor-5d4757c55b from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:41Z" name: success-200-isvc-2138e-predictor.18b32a50200ee006 namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: f0186244-8475-4ba8-86f7-893b474c1f41 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 72abf964-8ec5-4037-b5a6-471e7d6dd40a kind: Event lastTimestamp: "2026-05-26T16:27:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:26Z" name: success-200-isvc-2138e-predictor.18b32a539eddf1c4 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: 58e0ce76-7a69-4405-81e7-60436a3c6ae7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 72abf964-8ec5-4037-b5a6-471e7d6dd40a kind: Event lastTimestamp: "2026-05-26T16:27:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:26Z" name: success-200-isvc-2138e-predictor.18b32a539f413ff6 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: cad84722-3d47-4ef5-ac5d-89f23ce47b86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 72abf964-8ec5-4037-b5a6-471e7d6dd40a kind: Event lastTimestamp: "2026-05-26T16:27:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:56Z" name: success-200-isvc-2138e-predictor.18b32a5e1b6271f5 namespace: kserve-ci-e2e-test resourceVersion: "22318" uid: a39d7721-766a-43ff-991d-23c1851855e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2138e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 72abf964-8ec5-4037-b5a6-471e7d6dd40a kind: Event lastTimestamp: "2026-05-26T16:27:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:56Z" name: success-200-isvc-2138e-predictor.18b32a5e1bc82221 namespace: kserve-ci-e2e-test resourceVersion: "22319" uid: d9d3d559-65ef-478d-ba74-b695f075b46b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2138e namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: b6e4ab36-32fe-4672-b92f-0c0594ece35f kind: Event lastTimestamp: "2026-05-26T16:27:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:29Z" name: success-200-isvc-2138e.18b32a502015b4a0 namespace: kserve-ci-e2e-test resourceVersion: "22053" uid: ff41a004-ed2f-4ea7-9832-149428a5f5f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2138e namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: b6e4ab36-32fe-4672-b92f-0c0594ece35f kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-2138e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2138e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:41Z" name: success-200-isvc-2138e.18b32a50208fb2ff namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: aaf5cc87-4b43-4510-bb4e-7a2d59192a41 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2138e namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: b6e4ab36-32fe-4672-b92f-0c0594ece35f kind: Event lastTimestamp: "2026-05-26T16:26:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2138e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:41Z" name: success-200-isvc-2138e.18b32a50208fefe1 namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 93232d03-6f5d-41d6-90e4-31ae544fe077 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2138e namespace: kserve-ci-e2e-test resourceVersion: "22050" uid: b6e4ab36-32fe-4672-b92f-0c0594ece35f kind: Event lastTimestamp: "2026-05-26T16:27:29Z" message: InferenceService [success-200-isvc-2138e] is Ready metadata: creationTimestamp: "2026-05-26T16:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:29Z" name: success-200-isvc-2138e.18b32a5b5512bbba namespace: kserve-ci-e2e-test resourceVersion: "22051" uid: f3608f0c-dda3-4580-ae64-fc1fff3cbecf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:37:06.599710Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5798a-predictor-7b4956db94-97vx2 to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:37:06Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32ae1b2f276d5 namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: 0d390024-903a-47d4-89ca-2b514749ffb6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26941" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:37:07Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32ae1cca09fde namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: a4320347-fbf4-49be-87a5-5dbff6e342d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32ae1cdf7208a namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: a18ebdb4-c1f5-4ef6-932a-dc7bacb9f818 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32ae1d0b0468d namespace: kserve-ci-e2e-test resourceVersion: "26986" uid: 06bbb0b0-b618-4f1f-be03-b9a49d31a214 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32ae1d11d8557 namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: 6cb59207-e1e4-437b-9029-319f567016c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32ae1d12a38f8 namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: 903958a1-62df-49b8-93fc-da144f592e5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32ae1d4eb4433 namespace: kserve-ci-e2e-test resourceVersion: "26992" uid: 939d2277-74f2-41c0-9964-376ff152d5cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:37:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:07Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32ae1d5599602 namespace: kserve-ci-e2e-test resourceVersion: "26993" uid: b1d4910c-0645-455c-a1d2-b0c5c705cc31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:37:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:37:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:37:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:44Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32ae22893604d namespace: kserve-ci-e2e-test resourceVersion: "27369" uid: 13338aac-2471-4e0c-980f-56b36cc8d554 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:46:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:46:21Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32b62e060bb21 namespace: kserve-ci-e2e-test resourceVersion: "31047" uid: 950c6c5f-6183-43ed-9404-456ae595e6c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5798a-predictor-7b4956db94-97vx2 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 916c4746-0b07-4f1b-a54d-f5f8a961e19f kind: Event lastTimestamp: "2026-05-26T16:46:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:46:21Z" name: success-200-isvc-5798a-predictor-7b4956db94-97vx2.18b32b62e06147c7 namespace: kserve-ci-e2e-test resourceVersion: "31052" uid: c42bed6b-5af8-4b38-92d7-8d625d847890 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5798a-predictor-7b4956db94 namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: 7edcedd3-5beb-45a0-a227-974a5fb31b30 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'Created pod: success-200-isvc-5798a-predictor-7b4956db94-97vx2' metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:06Z" name: success-200-isvc-5798a-predictor-7b4956db94.18b32ae1b2861e04 namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 41eaea3c-31af-499c-a910-d8d6d65bd565 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: 88c673cb-80ee-4bc7-a4e7-c89ff8c176da kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: Scaled up replica set success-200-isvc-5798a-predictor-7b4956db94 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:06Z" name: success-200-isvc-5798a-predictor.18b32ae1b0a9c728 namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 7e03598b-980f-4ec5-955b-ecccef37c057 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: 9e8ea910-a9a1-48d9-8386-d6b40348cee6 kind: Event lastTimestamp: "2026-05-26T16:37:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:21Z" name: success-200-isvc-5798a-predictor.18b32ae52f20f4bc namespace: kserve-ci-e2e-test resourceVersion: "27129" uid: 9534da45-5666-4f47-b5d3-b5c15680da76 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: 9e8ea910-a9a1-48d9-8386-d6b40348cee6 kind: Event lastTimestamp: "2026-05-26T16:37:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:21Z" name: success-200-isvc-5798a-predictor.18b32ae52f86ff75 namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: 46a0eae9-4c29-4d6a-9f37-95e112b539e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27130" uid: 9e8ea910-a9a1-48d9-8386-d6b40348cee6 kind: Event lastTimestamp: "2026-05-26T16:38:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:21Z" name: success-200-isvc-5798a-predictor.18b32ae8ae1fd519 namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 4cd6077c-ef44-47f1-ad75-31d9d20841fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5798a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27130" uid: 9e8ea910-a9a1-48d9-8386-d6b40348cee6 kind: Event lastTimestamp: "2026-05-26T16:38:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:21Z" name: success-200-isvc-5798a-predictor.18b32ae8ae803d8d namespace: kserve-ci-e2e-test resourceVersion: "27716" uid: 35ebb00f-3b67-4dbe-bebb-a89c4bcf7913 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5798a namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: 4953301a-04bf-46a0-a955-310257781c7e kind: Event lastTimestamp: "2026-05-26T16:37:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:54Z" name: success-200-isvc-5798a.18b32ae1b07b9923 namespace: kserve-ci-e2e-test resourceVersion: "27463" uid: d7715914-1e74-42fa-be1e-a906ce6aac97 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5798a namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: 4953301a-04bf-46a0-a955-310257781c7e kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'Failed to update status for InferenceService "success-200-isvc-5798a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5798a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:06Z" name: success-200-isvc-5798a.18b32ae1b0ce189d namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 6592e7ed-52b5-45a4-bd87-3a9eb4766216 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5798a namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: 4953301a-04bf-46a0-a955-310257781c7e kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5798a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:06Z" name: success-200-isvc-5798a.18b32ae1b0ce5a22 namespace: kserve-ci-e2e-test resourceVersion: "26919" uid: ee2f8fa1-8509-43da-9445-9a53e6198050 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5798a namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 4953301a-04bf-46a0-a955-310257781c7e kind: Event lastTimestamp: "2026-05-26T16:37:54Z" message: InferenceService [success-200-isvc-5798a] is Ready metadata: creationTimestamp: "2026-05-26T16:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:54Z" name: success-200-isvc-5798a.18b32aece365f823 namespace: kserve-ci-e2e-test resourceVersion: "27461" uid: 439a108b-af94-400e-be9b-c0e606caadf3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:35:56.376324Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ad1594ea724 namespace: kserve-ci-e2e-test resourceVersion: "26047" uid: 103e9aaa-3b79-4f42-8473-39e22a2817d1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ad173261f50 namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: 9a68f2b4-f1f8-4869-9f0f-97f2c6134361 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ad174a0b5b0 namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: cbf804a5-44b4-4bc0-b1a5-af9987c57c70 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ad177da9852 namespace: kserve-ci-e2e-test resourceVersion: "26091" uid: 24447819-4123-4f7e-aba9-ed104ea13bee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ad1784d6440 namespace: kserve-ci-e2e-test resourceVersion: "26093" uid: a81eae99-0ebf-4d74-b6ff-90f2f4cfeb3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ad17859cdc3 namespace: kserve-ci-e2e-test resourceVersion: "26095" uid: 875c04ab-ee40-4824-8edd-b0363a1356f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ad17bc4dfab namespace: kserve-ci-e2e-test resourceVersion: "26098" uid: d6d91b1c-0e79-46e3-a8e8-8c6fdd8e5012 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ad17c49308a namespace: kserve-ci-e2e-test resourceVersion: "26099" uid: 6ec4ef99-0030-4c67-a12a-3f27221bf73c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:36:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:34Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ad1cc37102d namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: 0a6fef3a-6a3f-41be-a3c3-3eb07d7d50db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:06Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ae1b0a4b9ee namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 5973c82a-4604-4094-93f2-bf4ef153aee6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:06Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ae1b0a527d3 namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: 38bd3f6c-bc84-4cd9-8ec2-ddb44b045e86 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: b675e0c3-2029-4f3e-9b8b-bdbec82b6161 kind: Event lastTimestamp: "2026-05-26T16:37:09Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:09Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99.18b32ae254727769 namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: 66639d2f-f51d-49fc-86ff-e3f805f37a64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-82fbc-predictor-76cd8c4d48 namespace: kserve-ci-e2e-test resourceVersion: "26020" uid: 7139bf98-7f72-4e2f-a1aa-fe734a51e17f kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: 'Created pod: success-200-isvc-82fbc-predictor-76cd8c4d48-88z99' metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor-76cd8c4d48.18b32ad158bfa12f namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: fd180ee5-14e0-42ce-9964-60e2e4b6518c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-82fbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26015" uid: 927b93b1-6a08-4419-9fd5-dda555775627 kind: Event lastTimestamp: "2026-05-26T16:35:56Z" message: Scaled up replica set success-200-isvc-82fbc-predictor-76cd8c4d48 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:35:56Z" name: success-200-isvc-82fbc-predictor.18b32ad155262e30 namespace: kserve-ci-e2e-test resourceVersion: "26026" uid: 9057b810-342c-407f-b403-8fc302921a09 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-82fbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26018" uid: d62d309d-63b4-4e8d-94bf-bd833dce7047 kind: Event lastTimestamp: "2026-05-26T16:36:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:56Z" name: success-200-isvc-82fbc-predictor.18b32ad4d399e5cb namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: b56f1e12-fd90-4694-bbba-6baeb52714dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-82fbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26018" uid: d62d309d-63b4-4e8d-94bf-bd833dce7047 kind: Event lastTimestamp: "2026-05-26T16:36:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:56Z" name: success-200-isvc-82fbc-predictor.18b32ad4d427ab14 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 9c6df4f2-cf6f-423c-807a-52f92c3f038d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-82fbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26230" uid: d62d309d-63b4-4e8d-94bf-bd833dce7047 kind: Event lastTimestamp: "2026-05-26T16:36:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:26Z" name: success-200-isvc-82fbc-predictor.18b32ad852cba43d namespace: kserve-ci-e2e-test resourceVersion: "26338" uid: 8886d736-f005-4cc9-85a6-3b9908db5a63 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-82fbc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26230" uid: d62d309d-63b4-4e8d-94bf-bd833dce7047 kind: Event lastTimestamp: "2026-05-26T16:36:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:26Z" name: success-200-isvc-82fbc-predictor.18b32ad8533374d6 namespace: kserve-ci-e2e-test resourceVersion: "26340" uid: d399dca1-d527-44c0-8d89-6588886a05ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:35:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-82fbc namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: bc6ccc00-12e1-48b6-ba20-f3d495c832d6 kind: Event lastTimestamp: "2026-05-26T16:36:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:36:44Z" name: success-200-isvc-82fbc.18b32ad154fa6f3b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 5fd2ca4c-c61b-48fe-a45a-f8d8854bb6f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-82fbc namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: bc6ccc00-12e1-48b6-ba20-f3d495c832d6 kind: Event lastTimestamp: "2026-05-26T16:36:44Z" message: InferenceService [success-200-isvc-82fbc] is Ready metadata: creationTimestamp: "2026-05-26T16:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:36:44Z" name: success-200-isvc-82fbc.18b32adc87630b21 namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 21978b58-7e03-4acf-9b2e-a0d92e221000 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:27:21.169415Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-86373-predictor-59f94db984-rq2zd to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:27:21Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a5964950e38 namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 8febb95e-e444-41b4-aca1-6467e2712426 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-86373-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:21Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a597220cfed namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: 1f7e44fe-f473-46a2-8438-6d16bbeef3d5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21903" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:27:22Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a59a2556045 namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: 2616d564-112b-4246-811c-69ac5edd0d87 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a59a3abccdb namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 8f07219c-80e2-410f-9675-b25f72789712 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a59a70561d1 namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: 573f4c66-6421-4531-8c87-dbe9f1edf89e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a59a776ff21 namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 6fcd24fe-a355-48b5-b0e9-45f450823a99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a59a7849a64 namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: bb536aa5-5db8-49c5-be49-87ce8a8d69cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a59ab169d67 namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: 77d2415f-5a6b-432b-8e34-acd8a6860843 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:27:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:22Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a59ab8b8c13 namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 855314af-d846-4b1b-a8a2-9d14f807e1fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:36:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:38Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32a59b77c717b namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 30bef136-274b-4f9e-b6f8-a01e9eebe444 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:36Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32ada96efa8e2 namespace: kserve-ci-e2e-test resourceVersion: "26483" uid: a145241e-b9a5-4ae3-8ed2-43cd1d5808a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:36Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32ada96f0d445 namespace: kserve-ci-e2e-test resourceVersion: "26486" uid: 186a95cb-2321-40e7-ab87-a9ce617df8e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-86373-predictor-59f94db984-rq2zd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: fffa82fa-f5e1-4b1f-a032-1145a6df65f4 kind: Event lastTimestamp: "2026-05-26T16:36:38Z" message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp 10.133.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:38Z" name: success-200-isvc-86373-predictor-59f94db984-rq2zd.18b32adb2bd44592 namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: 682774d8-4aaf-4033-bb33-a366edfd0239 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-86373-predictor-59f94db984 namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: dd5a2f58-c1be-4284-8c9c-285db437b97f kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: 'Created pod: success-200-isvc-86373-predictor-59f94db984-rq2zd' metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:21Z" name: success-200-isvc-86373-predictor-59f94db984.18b32a59642d4bbd namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: 91ea8560-cdc3-4308-b22d-8a4bfe7d8b00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: 4d7106c7-e28a-4c11-882c-2e789b20b9ee kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: Scaled up replica set success-200-isvc-86373-predictor-59f94db984 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:21Z" name: success-200-isvc-86373-predictor.18b32a596263c6bd namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: 576ca7e6-5c09-4f23-aff4-2e5f4c1b53e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:27:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "21872" uid: 42f38987-0824-46f4-bff3-8413cc29e411 kind: Event lastTimestamp: "2026-05-26T16:28:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:06Z" name: success-200-isvc-86373-predictor.18b32a5ce128b786 namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 5d861f80-82ae-48cb-b0af-8a6d53523153 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:27:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "21872" uid: 42f38987-0824-46f4-bff3-8413cc29e411 kind: Event lastTimestamp: "2026-05-26T16:28:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:06Z" name: success-200-isvc-86373-predictor.18b32a5ce18b178c namespace: kserve-ci-e2e-test resourceVersion: "22398" uid: 600e282d-6dda-47a9-941b-a9e49f997449 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:27:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: 42f38987-0824-46f4-bff3-8413cc29e411 kind: Event lastTimestamp: "2026-05-26T16:28:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:27:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:36Z" name: success-200-isvc-86373-predictor.18b32a6060553c8a namespace: kserve-ci-e2e-test resourceVersion: "22676" uid: 116f6c00-484f-40f9-8ef6-462e4a694067 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:27:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-86373-predictor namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: 42f38987-0824-46f4-bff3-8413cc29e411 kind: Event lastTimestamp: "2026-05-26T16:28:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:27:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:28:36Z" name: success-200-isvc-86373-predictor.18b32a6060c2e639 namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: aeb47771-fee3-4c87-ac53-fa3649313cb2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-86373 namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 18034bdb-6f7f-4b2f-ae51-135c546b55cb kind: Event lastTimestamp: "2026-05-26T16:28:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:28:08Z" name: success-200-isvc-86373.18b32a5962419d4e namespace: kserve-ci-e2e-test resourceVersion: "22424" uid: 0ef6ab1c-1239-447f-85d4-8cc0ffedad9d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-86373 namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 18034bdb-6f7f-4b2f-ae51-135c546b55cb kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-86373": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-86373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:21Z" name: success-200-isvc-86373.18b32a59628a8750 namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: 38e3fb7c-18db-40a6-a012-b00df1159662 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-86373 namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 18034bdb-6f7f-4b2f-ae51-135c546b55cb kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-86373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:27:21Z" name: success-200-isvc-86373.18b32a59628ad633 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: 371cabb9-8866-43d8-8d3f-4ab4fdc39798 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:28:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-86373 namespace: kserve-ci-e2e-test resourceVersion: "22422" uid: 18034bdb-6f7f-4b2f-ae51-135c546b55cb kind: Event lastTimestamp: "2026-05-26T16:28:08Z" message: InferenceService [success-200-isvc-86373] is Ready metadata: creationTimestamp: "2026-05-26T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:28:08Z" name: success-200-isvc-86373.18b32a6472b4b4a1 namespace: kserve-ci-e2e-test resourceVersion: "22423" uid: 314c87b1-03a8-42b3-99d9-dd3a37b1e7db reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:26:10.940056Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:26:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:26:10Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a490a9612df namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 897d5ae4-8ea5-4576-b62d-4a9b65c0471d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:26:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a007a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:11Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a4911f48460 namespace: kserve-ci-e2e-test resourceVersion: "20954" uid: 4fa7a795-4bea-4b9c-93a1-daccefebf4b4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:26:11Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:26:11Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a494811af0f namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: 7b3ae245-80be-4582-8f41-89423b018c41 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:26:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:11Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a4949895bc3 namespace: kserve-ci-e2e-test resourceVersion: "20972" uid: a710481d-4d30-4c25-bc7c-e67c87d89cc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a494cb78079 namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: 9505db75-a16a-4ac3-aced-769bb3270cb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a494d304d21 namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 809f0579-f411-447c-9c41-7c1c4c1a43c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a494d3d9797 namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: aecb0fea-68e2-418c-a5f2-2da90e1db7a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a495055d1c8 namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 21c8168c-de57-4058-8f6f-f7c173056855 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:26:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:12Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a4950c76b33 namespace: kserve-ci-e2e-test resourceVersion: "20984" uid: ab5b62d9-719f-4110-bd32-b33fdecc8bee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:26:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:26:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:49Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a49978cd85e namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: a760f26f-5801-4468-aa6c-357f6989c681 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:21Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a5962b62408 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: 3caf2a72-be1e-4127-9413-5152952708de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:27:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:21Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a5962b677c9 namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: cb54ae37-1b1c-4030-b2e6-91f2e720c8f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4 namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 168729a0-3ea9-4f06-a196-fec2df57a92c kind: Event lastTimestamp: "2026-05-26T16:27:25Z" message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-05-26T16:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:27:25Z" name: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4.18b32a5a5b5b2f5e namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 60e8da14-d8c0-4dbf-901e-9301acdaee12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a007a-predictor-5f5fc59776 namespace: kserve-ci-e2e-test resourceVersion: "20891" uid: d1acaaeb-4585-4703-8a96-a750c173e0ba kind: Event lastTimestamp: "2026-05-26T16:26:10Z" message: 'Created pod: success-200-isvc-a007a-predictor-5f5fc59776-qcnp4' metadata: creationTimestamp: "2026-05-26T16:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:10Z" name: success-200-isvc-a007a-predictor-5f5fc59776.18b32a490a343bab namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: 84f20c41-a6d7-4496-9fb6-6d0d722d7c6a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 38a52fa1-3cc5-409d-8d03-11c0800cb66e kind: Event lastTimestamp: "2026-05-26T16:26:10Z" message: Scaled up replica set success-200-isvc-a007a-predictor-5f5fc59776 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:10Z" name: success-200-isvc-a007a-predictor.18b32a4908597dc5 namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: 7738828d-cfdb-4992-a280-fdd558c97608 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:26:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: afec41d9-ed20-44fd-9da0-f31a3ba88b43 kind: Event lastTimestamp: "2026-05-26T16:26:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:55Z" name: success-200-isvc-a007a-predictor.18b32a4c86ff7c06 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: f16b0b55-725a-4406-aa83-9c170f161128 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:26:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: afec41d9-ed20-44fd-9da0-f31a3ba88b43 kind: Event lastTimestamp: "2026-05-26T16:26:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:26:55Z" name: success-200-isvc-a007a-predictor.18b32a4c87c0fb67 namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: 5009721b-6416-49fc-9ac5-3aac87eeceba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: afec41d9-ed20-44fd-9da0-f31a3ba88b43 kind: Event lastTimestamp: "2026-05-26T16:27:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:10Z" name: success-200-isvc-a007a-predictor.18b32a570480ab51 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 478e2cd9-eebb-49c7-b5f2-054685edf564 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a007a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: afec41d9-ed20-44fd-9da0-f31a3ba88b43 kind: Event lastTimestamp: "2026-05-26T16:27:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:27:10Z" name: success-200-isvc-a007a-predictor.18b32a5704e16aec namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: 3b83cf40-9f63-4611-9ec2-5b1eeb99a250 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a007a namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: 5e72a7dd-f6b2-4a26-8c80-45c5a45b20b5 kind: Event lastTimestamp: "2026-05-26T16:26:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:59Z" name: success-200-isvc-a007a.18b32a490863162b namespace: kserve-ci-e2e-test resourceVersion: "21610" uid: 8701429a-3245-4ed6-b558-8a25ed09ce9c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a007a namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: 5e72a7dd-f6b2-4a26-8c80-45c5a45b20b5 kind: Event lastTimestamp: "2026-05-26T16:26:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-a007a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a007a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:10Z" name: success-200-isvc-a007a.18b32a4908e63f00 namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: 97367afe-2b66-4e1b-9fbf-c13eca82c93a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a007a namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: 5e72a7dd-f6b2-4a26-8c80-45c5a45b20b5 kind: Event lastTimestamp: "2026-05-26T16:26:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a007a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:10Z" name: success-200-isvc-a007a.18b32a4908e68861 namespace: kserve-ci-e2e-test resourceVersion: "20899" uid: 6d1b8c9a-bc25-4f8f-af89-b372a0e4b722 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a007a namespace: kserve-ci-e2e-test resourceVersion: "21605" uid: 5e72a7dd-f6b2-4a26-8c80-45c5a45b20b5 kind: Event lastTimestamp: "2026-05-26T16:26:59Z" message: InferenceService [success-200-isvc-a007a] is Ready metadata: creationTimestamp: "2026-05-26T16:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:26:59Z" name: success-200-isvc-a007a.18b32a5453adeb7c namespace: kserve-ci-e2e-test resourceVersion: "21607" uid: ee659a49-0ac7-4c3d-b088-185c90fe0d64 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:16:56.055717Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:16:56Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c7d8e70316 namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: ce8bcc73-6908-4789-8477-91260c48a965 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ad62d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:56Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c7e5106027 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 8167e3c8-5473-4c06-b70e-dee58503b099 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:16:57Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c81639740f namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 1b015a8e-b853-4abc-8b7d-44e902f313be reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c817817f1b namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: a8d4a85a-3627-498f-80ff-572fe45ee65b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c81a4bf731 namespace: kserve-ci-e2e-test resourceVersion: "16699" uid: 58e24823-2c15-44aa-999e-0509b0fab03e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c81abe271a namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: 38c08680-98ed-424e-88da-eb4fcb584f8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c81aca2bf8 namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: c8d33a6c-449d-4188-89d4-910f9246491b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c81dcc5507 namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: 673969dc-d11c-4bd0-9337-ef910e8d08a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:16:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:57Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c81e422e0b namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: c1bd4e8e-99d5-4986-9d95-8fadf97e5b3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:16:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:17:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:34Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b329c8646fb44b namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 0f43d401-1476-494d-a0b5-ddcd062bc198 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:26:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:10Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b32a49093b7bef namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: 2620a921-15b4-4274-91b8-650676b124b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: ce4a1f89-b0f7-46c3-b82a-537682b708c6 kind: Event lastTimestamp: "2026-05-26T16:26:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:26:10Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68.18b32a49093c0982 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 081c98d9-6333-4957-927c-ad30e49ec987 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ad62d-predictor-75c98cb7b4 namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: fb668fcd-4e88-4f3d-b18d-39a90f5a6e20 kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: 'Created pod: success-200-isvc-ad62d-predictor-75c98cb7b4-6gj68' metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:56Z" name: success-200-isvc-ad62d-predictor-75c98cb7b4.18b329c7d86ca871 namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 8ebbeccf-9354-42ad-88ee-cf0c3ac81b0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: e1df09ec-6b46-401c-b0da-74a7b5cb025c kind: Event lastTimestamp: "2026-05-26T16:16:56Z" message: Scaled up replica set success-200-isvc-ad62d-predictor-75c98cb7b4 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:56Z" name: success-200-isvc-ad62d-predictor.18b329c7d6a1b9f4 namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: b1627ed4-a11d-4d35-9285-03bb80a125b7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:17:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: 0b916567-0c3b-414d-81eb-dd274b46f961 kind: Event lastTimestamp: "2026-05-26T16:17:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:56Z" name: success-200-isvc-ad62d-predictor.18b329cb55162f3d namespace: kserve-ci-e2e-test resourceVersion: "17291" uid: 92f79baa-1b90-4664-865d-f846becd3719 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T16:17:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: 0b916567-0c3b-414d-81eb-dd274b46f961 kind: Event lastTimestamp: "2026-05-26T16:17:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:56Z" name: success-200-isvc-ad62d-predictor.18b329cb55a511d0 namespace: kserve-ci-e2e-test resourceVersion: "17294" uid: 22848134-827f-49b6-833f-6c05cd6ab8ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:18:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16874" uid: 0b916567-0c3b-414d-81eb-dd274b46f961 kind: Event lastTimestamp: "2026-05-26T16:18:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:26Z" name: success-200-isvc-ad62d-predictor.18b329d950de485f namespace: kserve-ci-e2e-test resourceVersion: "17537" uid: 9c2bb5ce-2813-49ef-89d4-d00d4d008d95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:18:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ad62d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16874" uid: 0b916567-0c3b-414d-81eb-dd274b46f961 kind: Event lastTimestamp: "2026-05-26T16:18:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:18:26Z" name: success-200-isvc-ad62d-predictor.18b329d9513ed99b namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 061ed63d-7dfc-4ccb-aee2-502a672b2347 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:16:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ad62d namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: 506808a7-3136-41cf-bcb2-f1bcc0919b88 kind: Event lastTimestamp: "2026-05-26T16:17:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:44Z" name: success-200-isvc-ad62d.18b329c7d67d91c7 namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 8a3b6a5d-e086-44c2-a948-98cea9ad4b5c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ad62d namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: 506808a7-3136-41cf-bcb2-f1bcc0919b88 kind: Event lastTimestamp: "2026-05-26T16:17:44Z" message: InferenceService [success-200-isvc-ad62d] is Ready metadata: creationTimestamp: "2026-05-26T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:44Z" name: success-200-isvc-ad62d.18b329d31ef6e8ff namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: ee12c358-eaf7-4d10-a9ec-341c049269ea reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:36:36.069338Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:36:36Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32ada9731dff4 namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: f75435c7-a454-40ae-9cc1-081fa23251f9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-bffff-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:36Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32adaa235f974 namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: 7364f9b5-75b4-4825-a87d-2d5d00818e06 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:36:37Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:36:37Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32adad49b61f5 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 0e43ac15-c6f0-45d6-a4fd-a897457e7470 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:36:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:37Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32adad62505ae namespace: kserve-ci-e2e-test resourceVersion: "26552" uid: f46cd0c4-0448-4d16-a97e-f22dd3e9704f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:36:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:37Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32adad9cb4475 namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: b47319b5-12b5-4310-a836-33c89efce0f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:36:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:37Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32adada57879c namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 6e0d3a32-2aa9-422b-bf59-7899f5fde91d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:36:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:37Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32adada655ac3 namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: 0f018428-7800-4562-b470-1c9291096302 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:36:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:37Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32adadd8eaa3a namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: 5cf23785-549c-4f85-b7cb-8788e6f60d7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:36:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:36:37Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32adade06b6d2 namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: 01ceca68-14d0-40a2-bf15-3928bbbcde0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:37:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:13Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32adae9e290ce namespace: kserve-ci-e2e-test resourceVersion: "27059" uid: 7eedfa3b-7d5b-4827-a18a-2053ed037796 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:53:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:53:55Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32bcc9ad498cf namespace: kserve-ci-e2e-test resourceVersion: "34204" uid: 9c52c881-eebc-47cf-8d22-71ce4fa71dd7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:53:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:53:55Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32bcc9ad52d97 namespace: kserve-ci-e2e-test resourceVersion: "34205" uid: d4e26931-4140-4e48-ab38-32c3603ecb71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:53:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: cde33cb9-e03d-4fae-9ce5-565ba7e7c5d2 kind: Event lastTimestamp: "2026-05-26T16:53:58Z" message: 'Readiness probe failed: Get "https://10.133.0.45:8643/healthz": dial tcp 10.133.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:53:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:53:58Z" name: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj.18b32bcd4a8c8106 namespace: kserve-ci-e2e-test resourceVersion: "34262" uid: 42c32723-c9c0-4ca1-8099-b0c4a189a4b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bffff-predictor-6cc9bbd78 namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 0a4e1e7b-b07b-4630-b33a-8577032f8f5e kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: 'Created pod: success-200-isvc-bffff-predictor-6cc9bbd78-2lwkj' metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:36Z" name: success-200-isvc-bffff-predictor-6cc9bbd78.18b32ada965ac511 namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: 0a469d6f-0c18-47d5-a72b-ae61b2080a4e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: f08c8cc4-518b-40e6-a081-cd1e7f8c09ea kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: Scaled up replica set success-200-isvc-bffff-predictor-6cc9bbd78 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:36:36Z" name: success-200-isvc-bffff-predictor.18b32ada941980b1 namespace: kserve-ci-e2e-test resourceVersion: "26465" uid: d6704510-9bfd-4898-b1d4-2b5460744593 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:36:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: 054394f4-6c97-4317-8ec5-60eb440081a9 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:06Z" name: success-200-isvc-bffff-predictor.18b32ade12a04b07 namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 0a4bbcdc-1ad2-4972-801f-42a65a4f0ed9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:36:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: 054394f4-6c97-4317-8ec5-60eb440081a9 kind: Event lastTimestamp: "2026-05-26T16:37:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:06Z" name: success-200-isvc-bffff-predictor.18b32ade13064bc9 namespace: kserve-ci-e2e-test resourceVersion: "26859" uid: a1974ef7-38c6-440b-9c1f-0682d3b99706 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: 054394f4-6c97-4317-8ec5-60eb440081a9 kind: Event lastTimestamp: "2026-05-26T16:37:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:51Z" name: success-200-isvc-bffff-predictor.18b32ae510b5c6bd namespace: kserve-ci-e2e-test resourceVersion: "27417" uid: 5f56c289-cb35-4679-917e-ade3c08858c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bffff-predictor namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: 054394f4-6c97-4317-8ec5-60eb440081a9 kind: Event lastTimestamp: "2026-05-26T16:37:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:51Z" name: success-200-isvc-bffff-predictor.18b32ae511159c97 namespace: kserve-ci-e2e-test resourceVersion: "27418" uid: 910f3e2f-d037-46c1-92f3-c9c0cabf3f32 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bffff namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: ce3153b1-b04e-4aae-8883-888810dfb2ab kind: Event lastTimestamp: "2026-05-26T16:37:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:23Z" name: success-200-isvc-bffff.18b32ada9406c6ca namespace: kserve-ci-e2e-test resourceVersion: "27158" uid: 33289586-136d-41e3-8bc2-bc8b82865d54 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bffff namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: ce3153b1-b04e-4aae-8883-888810dfb2ab kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-bffff": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bffff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:36:36Z" name: success-200-isvc-bffff.18b32ada9500181a namespace: kserve-ci-e2e-test resourceVersion: "26471" uid: aebc0606-7eb7-4e00-ae38-e56c208db1f7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:36:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bffff namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: ce3153b1-b04e-4aae-8883-888810dfb2ab kind: Event lastTimestamp: "2026-05-26T16:36:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bffff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:36:36Z" name: success-200-isvc-bffff.18b32ada95005e97 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1b293085-b93f-413c-9bed-d1f41cdd7526 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bffff namespace: kserve-ci-e2e-test resourceVersion: "27156" uid: ce3153b1-b04e-4aae-8883-888810dfb2ab kind: Event lastTimestamp: "2026-05-26T16:37:23Z" message: InferenceService [success-200-isvc-bffff] is Ready metadata: creationTimestamp: "2026-05-26T16:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:23Z" name: success-200-isvc-bffff.18b32ae5a6057b62 namespace: kserve-ci-e2e-test resourceVersion: "27157" uid: bc0332ea-a9a1-44f8-8d2f-68aa0a4ed274 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:14:45.680910Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c73f1-predictor-9985b79f8-6gnkh to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:14:45Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329a97df55896 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 11fb5abd-337b-4e69-9a70-f5f361132a46 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:14:46Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329a997a392a6 namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: ff3ffe2c-86f8-4abe-a673-254015cd27a6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:14:46Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T16:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:46Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329a9994c5376 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: 5ba08e35-cce4-46d3-b7e0-d79a629f67cb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:14:57Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" in 11.767s (11.767s including waiting). Image size: 1335714929 bytes.' metadata: creationTimestamp: "2026-05-26T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:57Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329ac56b82479 namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: f6b1f9c2-c34e-436a-be95-ca20148a037c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:14:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:57Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329ac5a35ff4a namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 65ff5417-89d0-4df6-a9f3-d63b4b2b4e35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:14:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:57Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329ac5aa7bab2 namespace: kserve-ci-e2e-test resourceVersion: "15278" uid: 8d834bad-bb00-41b3-ba2c-f47f02e6cad5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:14:57Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T16:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:14:57Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329ac5ab51c04 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 45f859a1-11a7-496f-819a-a0de1d0ecf9f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:15:00Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.286s (2.286s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T16:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:00Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329ace2f7d95f namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 8a6ca5c6-01f3-4388-8f61-865b1294ecdf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:15:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:00Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329ace66c0e83 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: e6a41dcd-8253-4c89-b851-936c47bd3bf3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:15:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:15:00Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329ace6d960fa namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: d43072aa-69c7-495c-a77c-68a7b8c7f8fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T16:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:17Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329ad497a0353 namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: 4e7e7120-0cc1-4780-a6fc-98d8ee61ec4b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329be8c426de1 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 914aec10-ee05-4768-b8c6-3560f2b38555 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329be8c42fc83 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: 3f03d19e-2ac1-45ac-a290-0c4c9022658f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: aa96e43d-98b3-4804-bf65-e705013a2e3c kind: Event lastTimestamp: "2026-05-26T16:16:17Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:17Z" name: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh.18b329befb9a234c namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: da51bb53-b259-492b-99ec-2b70307c7ff1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c73f1-predictor-9985b79f8 namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: 5c71ee3f-9143-47b8-9c68-587275798113 kind: Event lastTimestamp: "2026-05-26T16:14:45Z" message: 'Created pod: success-200-isvc-c73f1-predictor-9985b79f8-6gnkh' metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:45Z" name: success-200-isvc-c73f1-predictor-9985b79f8.18b329a97d8874df namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 8afea012-f8d2-4df1-90c6-50bbbe4d0c7e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: f1343a5f-124b-4bf7-870b-0c43bb3e99f7 kind: Event lastTimestamp: "2026-05-26T16:14:45Z" message: Scaled up replica set success-200-isvc-c73f1-predictor-9985b79f8 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:14:45Z" name: success-200-isvc-c73f1-predictor.18b329a97a675024 namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: 9595b12d-e30c-4c4d-8139-8f131ac3fc4d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:15:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: c95924ca-cadb-4c44-bc91-546bc0487307 kind: Event lastTimestamp: "2026-05-26T16:15:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:30Z" name: success-200-isvc-c73f1-predictor.18b329acffd9e53d namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: da417976-49e3-4187-8103-b942445c8cbe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:15:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: c95924ca-cadb-4c44-bc91-546bc0487307 kind: Event lastTimestamp: "2026-05-26T16:15:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:15:30Z" name: success-200-isvc-c73f1-predictor.18b329ad004447dc namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: f8b6b2af-c6d7-40b8-a96e-e2684efe25ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: c95924ca-cadb-4c44-bc91-546bc0487307 kind: Event lastTimestamp: "2026-05-26T16:16:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:15Z" name: success-200-isvc-c73f1-predictor.18b329b77cde5966 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: b40b3d35-e29a-4e4b-b584-719fb794d83f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c73f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: c95924ca-cadb-4c44-bc91-546bc0487307 kind: Event lastTimestamp: "2026-05-26T16:16:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:15Z" name: success-200-isvc-c73f1-predictor.18b329b77d723326 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 4757b167-4420-4e7b-ba97-91aa19342566 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:14:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c73f1 namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: c9af8d24-86c8-4f8d-9caf-d6c7b63c683e kind: Event lastTimestamp: "2026-05-26T16:15:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:15:48Z" name: success-200-isvc-c73f1.18b329a980a69eb5 namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: bf8bde8c-1787-4d3b-90af-626bce423f67 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c73f1 namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: c9af8d24-86c8-4f8d-9caf-d6c7b63c683e kind: Event lastTimestamp: "2026-05-26T16:14:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-c73f1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c73f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:14:45Z" name: success-200-isvc-c73f1.18b329a980f39856 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 880d5096-e455-44ad-b300-fed6d9b61e01 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:14:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c73f1 namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: c9af8d24-86c8-4f8d-9caf-d6c7b63c683e kind: Event lastTimestamp: "2026-05-26T16:14:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c73f1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:14:45Z" name: success-200-isvc-c73f1.18b329a980f3de86 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 1aac1d3b-8df5-430f-b748-a51fce5a3889 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:15:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c73f1 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: c9af8d24-86c8-4f8d-9caf-d6c7b63c683e kind: Event lastTimestamp: "2026-05-26T16:15:48Z" message: InferenceService [success-200-isvc-c73f1] is Ready metadata: creationTimestamp: "2026-05-26T16:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:15:48Z" name: success-200-isvc-c73f1.18b329b803dcfe58 namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 5b243405-a808-4a62-8429-ece38721b6e6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:16:16.154670Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-df0d2-predictor-8fddf9f46-js5fq to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b329be8e9d5046 namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: 3ea920b6-9672-44d8-925a-20d180a40cf3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b329bea8320dff namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: a33004d5-d1e1-4198-a81d-b06dbb9e1273 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b329bea9783e37 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 2c7d7749-c182-4d84-891b-0e8c7eed834c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b329beacff22a9 namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: a1b96b4d-bdc6-4e39-b8af-f38747634f73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b329bead824525 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: ddef8d19-fc05-4eee-bd66-efc9bacacf7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b329bead8ec942 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 5007ee0f-8f8f-4901-ac3f-d9bb3c9c8cb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b329beb1f12347 namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 3ba956ec-9e34-438c-954b-95ede967a022 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b329beb2665667 namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: 027a3be3-3c25-415f-b741-cf1fc82532ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:16:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:16:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T16:16:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:54Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b329bf0b8b3b75 namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: f8b92a87-13cb-4272-8055-40ddaefa6350 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b32a3fc1e65f7d namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 97eac00e-56c5-4198-99d5-9daafdca948a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:25:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T16:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:31Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b32a3fc1e6d22c namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: aac32df6-5b9b-4cd5-a8b8-2d7d7e56d84e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:25:35Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-05-26T16:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:35Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b32a40bb8ec0d5 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: 809b3e09-7981-4ef7-855b-aab6f52735f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: a1d29b3a-27a7-4483-9a6a-afe511aebef2 kind: Event lastTimestamp: "2026-05-26T16:25:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: i/o timeout' metadata: creationTimestamp: "2026-05-26T16:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:35Z" name: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq.18b32a40bbdf285a namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 12c98ebf-fad4-4df6-b429-0ddae56e1ec6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-df0d2-predictor-8fddf9f46 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: b77be561-7456-4aaa-a1cb-579b76043865 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: 'Created pod: success-200-isvc-df0d2-predictor-8fddf9f46-js5fq' metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor-8fddf9f46.18b329be8e176b2e namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 6f322b8f-20d3-477b-8042-8414a4eab6d1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 807737d8-8fc6-4f79-8e86-61d372793297 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: Scaled up replica set success-200-isvc-df0d2-predictor-8fddf9f46 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2-predictor.18b329be8c8e85dd namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: a8688693-be09-4f1b-91ea-7bcd397080f9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 1d457bdf-4bea-461a-8d3a-7e07995127d3 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:46Z" name: success-200-isvc-df0d2-predictor.18b329c20af1fdf4 namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: e72e1745-4c31-4f7e-9fb1-3a9711b9fdca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 1d457bdf-4bea-461a-8d3a-7e07995127d3 kind: Event lastTimestamp: "2026-05-26T16:16:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:46Z" name: success-200-isvc-df0d2-predictor.18b329c20b51a2d1 namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 0c591253-804e-4dd1-9192-0391f94367ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: 1d457bdf-4bea-461a-8d3a-7e07995127d3 kind: Event lastTimestamp: "2026-05-26T16:17:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:31Z" name: success-200-isvc-df0d2-predictor.18b329c9094b7738 namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: 6f539d46-fce0-49b2-9977-7edab53fa927 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T16:17:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: 1d457bdf-4bea-461a-8d3a-7e07995127d3 kind: Event lastTimestamp: "2026-05-26T16:17:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T16:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:31Z" name: success-200-isvc-df0d2-predictor.18b329c909b4916f namespace: kserve-ci-e2e-test resourceVersion: "17072" uid: 1b516c40-6f67-4150-9bb0-222c9d88a464 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 190061c2-5fb4-487a-bf54-1ced7cb6b744 kind: Event lastTimestamp: "2026-05-26T16:17:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:04Z" name: success-200-isvc-df0d2.18b329be8c6a6931 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 4b5f60a3-dee9-413d-9513-b10c5e3b7eee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 190061c2-5fb4-487a-bf54-1ced7cb6b744 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-df0d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2.18b329be8cb5b029 namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: cf007d20-219b-43bf-b5ad-08695324aa9a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 190061c2-5fb4-487a-bf54-1ced7cb6b744 kind: Event lastTimestamp: "2026-05-26T16:16:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T16:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:16Z" name: success-200-isvc-df0d2.18b329be8cb5f552 namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: a272e846-fa6c-4384-8679-845eaef5ec3b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16805" uid: 190061c2-5fb4-487a-bf54-1ced7cb6b744 kind: Event lastTimestamp: "2026-05-26T16:17:04Z" message: InferenceService [success-200-isvc-df0d2] is Ready metadata: creationTimestamp: "2026-05-26T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:04Z" name: success-200-isvc-df0d2.18b329c9c67d682a namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: 56da797a-00ce-48de-9d33-ec294b516b39 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:37:36.151142Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bffff-7994c5b4b7-cdjp9 namespace: kserve-ci-e2e-test resourceVersion: "27273" uid: 021f32c3-aaff-47f1-a253-83888421c2e6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-bffff-7994c5b4b7-cdjp9 to ip-10-0-137-193.ec2.internal metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:37:36Z" name: switch-graph-bffff-7994c5b4b7-cdjp9.18b32ae8945982b4 namespace: kserve-ci-e2e-test resourceVersion: "27280" uid: 60741c1d-eef9-41aa-a5da-7e8877225e7e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bffff-7994c5b4b7-cdjp9 namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: 021f32c3-aaff-47f1-a253-83888421c2e6 kind: Event lastTimestamp: "2026-05-26T16:37:36Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:37:36Z" name: switch-graph-bffff-7994c5b4b7-cdjp9.18b32ae8aef43c31 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: c8aa6ca1-e899-4c15-a149-272480135a34 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bffff} kind: Pod name: switch-graph-bffff-7994c5b4b7-cdjp9 namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: 021f32c3-aaff-47f1-a253-83888421c2e6 kind: Event lastTimestamp: "2026-05-26T16:37:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:36Z" name: switch-graph-bffff-7994c5b4b7-cdjp9.18b32ae8b0528803 namespace: kserve-ci-e2e-test resourceVersion: "27300" uid: 47b0c088-fc1f-4df1-b7a0-68ea03e75654 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bffff} kind: Pod name: switch-graph-bffff-7994c5b4b7-cdjp9 namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: 021f32c3-aaff-47f1-a253-83888421c2e6 kind: Event lastTimestamp: "2026-05-26T16:37:36Z" message: 'Created container: switch-graph-bffff' metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:36Z" name: switch-graph-bffff-7994c5b4b7-cdjp9.18b32ae8b3f13bcb namespace: kserve-ci-e2e-test resourceVersion: "27303" uid: 6d544d76-65f0-44bb-84aa-61b30d0dc128 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bffff} kind: Pod name: switch-graph-bffff-7994c5b4b7-cdjp9 namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: 021f32c3-aaff-47f1-a253-83888421c2e6 kind: Event lastTimestamp: "2026-05-26T16:37:36Z" message: Started container switch-graph-bffff metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:37:36Z" name: switch-graph-bffff-7994c5b4b7-cdjp9.18b32ae8b467fbf9 namespace: kserve-ci-e2e-test resourceVersion: "27308" uid: 3572ea02-94dc-4a8f-9c4e-3715f365c4bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bffff} kind: Pod name: switch-graph-bffff-7994c5b4b7-cdjp9 namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: 021f32c3-aaff-47f1-a253-83888421c2e6 kind: Event lastTimestamp: "2026-05-26T16:53:55Z" message: Stopping container switch-graph-bffff metadata: creationTimestamp: "2026-05-26T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:53:55Z" name: switch-graph-bffff-7994c5b4b7-cdjp9.18b32bcc91e4e156 namespace: kserve-ci-e2e-test resourceVersion: "34176" uid: 01582b1c-3fb3-4faf-8689-692cf01e3b4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T16:53:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bffff} kind: Pod name: switch-graph-bffff-7994c5b4b7-cdjp9 namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: 021f32c3-aaff-47f1-a253-83888421c2e6 kind: Event lastTimestamp: "2026-05-26T16:54:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:53:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:54:18Z" name: switch-graph-bffff-7994c5b4b7-cdjp9.18b32bcd532f7077 namespace: kserve-ci-e2e-test resourceVersion: "34423" uid: 5957b489-611b-41a6-a9f7-a01dd299b25c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-193.ec2.internal source: component: kubelet host: ip-10-0-137-193.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-bffff-7994c5b4b7 namespace: kserve-ci-e2e-test resourceVersion: "27269" uid: 326bb1ee-4d1a-4080-9ebf-5c0b30ce1e11 kind: Event lastTimestamp: "2026-05-26T16:37:36Z" message: 'Created pod: switch-graph-bffff-7994c5b4b7-cdjp9' metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:36Z" name: switch-graph-bffff-7994c5b4b7.18b32ae8933c0a69 namespace: kserve-ci-e2e-test resourceVersion: "27276" uid: 71b9d2a8-9803-4796-959d-026df5509221 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-bffff namespace: kserve-ci-e2e-test resourceVersion: "27267" uid: c1232249-2f61-4779-9cfa-6bc6a16e3392 kind: Event lastTimestamp: "2026-05-26T16:37:36Z" message: Scaled up replica set switch-graph-bffff-7994c5b4b7 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:37:36Z" name: switch-graph-bffff.18b32ae8925dec49 namespace: kserve-ci-e2e-test resourceVersion: "27271" uid: a33a8727-6e85-40a9-8ed2-0930c9ef0ec5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:37:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-bffff namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: da61ba9b-2906-40df-8dbb-b176345b4ae3 kind: Event lastTimestamp: "2026-05-26T16:37:36Z" message: InferenceGraph [switch-graph-bffff] is Ready metadata: creationTimestamp: "2026-05-26T16:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:37:36Z" name: switch-graph-bffff.18b32ae8983bf50c namespace: kserve-ci-e2e-test resourceVersion: "27287" uid: 93c2e6cc-b98a-434a-bb29-a18422454d9b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bffff namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: b92a98b9-3616-45bf-ad65-b4077ccea08a kind: Event lastTimestamp: "2026-05-26T16:38:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:06Z" name: switch-graph-bffff.18b32aec10df549b namespace: kserve-ci-e2e-test resourceVersion: "27552" uid: 5f8796ae-3f00-4263-9527-27d51392e3a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bffff namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: b92a98b9-3616-45bf-ad65-b4077ccea08a kind: Event lastTimestamp: "2026-05-26T16:38:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:38:06Z" name: switch-graph-bffff.18b32aec11296f24 namespace: kserve-ci-e2e-test resourceVersion: "27553" uid: 1e8c5dbf-6f38-4721-84c4-f461224eb6d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:16:05.788271Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c73f1-864fbbd8d6-ns5f9 namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: 0e437aae-3787-4015-bd15-6d58f1c88265 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-c73f1-864fbbd8d6-ns5f9 to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:16:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:16:05Z" name: switch-graph-c73f1-864fbbd8d6-ns5f9.18b329bc24baa479 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 3eb26e19-ea2f-435f-ac0c-547b532a4ae0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:06Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c73f1-864fbbd8d6-ns5f9 namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 0e437aae-3787-4015-bd15-6d58f1c88265 kind: Event lastTimestamp: "2026-05-26T16:16:06Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:16:06Z" name: switch-graph-c73f1-864fbbd8d6-ns5f9.18b329bc3e4d7ae6 namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: f11b67e0-c15f-4d16-9d65-fd6e4e5255a6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c73f1} kind: Pod name: switch-graph-c73f1-864fbbd8d6-ns5f9 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 0e437aae-3787-4015-bd15-6d58f1c88265 kind: Event lastTimestamp: "2026-05-26T16:16:06Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" metadata: creationTimestamp: "2026-05-26T16:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:06Z" name: switch-graph-c73f1-864fbbd8d6-ns5f9.18b329bc3fabc790 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: fd866627-f56d-44be-bbc3-d95bbaf09607 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c73f1} kind: Pod name: switch-graph-c73f1-864fbbd8d6-ns5f9 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 0e437aae-3787-4015-bd15-6d58f1c88265 kind: Event lastTimestamp: "2026-05-26T16:16:08Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" in 2.387s (2.387s including waiting). Image size: 217997401 bytes.' metadata: creationTimestamp: "2026-05-26T16:16:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:08Z" name: switch-graph-c73f1-864fbbd8d6-ns5f9.18b329bcce00da16 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 26050e19-3e12-41da-82aa-90f7876d62bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c73f1} kind: Pod name: switch-graph-c73f1-864fbbd8d6-ns5f9 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 0e437aae-3787-4015-bd15-6d58f1c88265 kind: Event lastTimestamp: "2026-05-26T16:16:08Z" message: 'Created container: switch-graph-c73f1' metadata: creationTimestamp: "2026-05-26T16:16:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:08Z" name: switch-graph-c73f1-864fbbd8d6-ns5f9.18b329bcd16e2bee namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: eb6131f7-c137-4050-a9aa-b8cbda7cf9aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c73f1} kind: Pod name: switch-graph-c73f1-864fbbd8d6-ns5f9 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 0e437aae-3787-4015-bd15-6d58f1c88265 kind: Event lastTimestamp: "2026-05-26T16:16:08Z" message: Started container switch-graph-c73f1 metadata: creationTimestamp: "2026-05-26T16:16:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:08Z" name: switch-graph-c73f1-864fbbd8d6-ns5f9.18b329bcd1df9526 namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: 67e592e6-976f-475b-8ad2-5e54fb13e6e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c73f1} kind: Pod name: switch-graph-c73f1-864fbbd8d6-ns5f9 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 0e437aae-3787-4015-bd15-6d58f1c88265 kind: Event lastTimestamp: "2026-05-26T16:16:15Z" message: Stopping container switch-graph-c73f1 metadata: creationTimestamp: "2026-05-26T16:16:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:15Z" name: switch-graph-c73f1-864fbbd8d6-ns5f9.18b329be84dd9612 namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 3810e6cd-4e6f-40bc-8ba7-4e3b78c046fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:16:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c73f1} kind: Pod name: switch-graph-c73f1-864fbbd8d6-ns5f9 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 0e437aae-3787-4015-bd15-6d58f1c88265 kind: Event lastTimestamp: "2026-05-26T16:16:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:16:45Z" name: switch-graph-c73f1-864fbbd8d6-ns5f9.18b329bf81042f4d namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: 2d4b736f-16bc-42ab-9e11-40974c06e94e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-c73f1-864fbbd8d6 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: ff13657f-4de5-48e8-a747-8e6421591b68 kind: Event lastTimestamp: "2026-05-26T16:16:05Z" message: 'Created pod: switch-graph-c73f1-864fbbd8d6-ns5f9' metadata: creationTimestamp: "2026-05-26T16:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:05Z" name: switch-graph-c73f1-864fbbd8d6.18b329bc24493415 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 335751cf-05fc-4b89-8cc2-c52458daae2c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-c73f1 namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: 8cbc5f88-477a-49b5-93a3-582004fe23a7 kind: Event lastTimestamp: "2026-05-26T16:16:05Z" message: Scaled up replica set switch-graph-c73f1-864fbbd8d6 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:16:05Z" name: switch-graph-c73f1.18b329bc23823464 namespace: kserve-ci-e2e-test resourceVersion: "15906" uid: 9bb9b161-db7e-40c9-982f-39dc95c29efc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:16:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-c73f1 namespace: kserve-ci-e2e-test resourceVersion: "15919" uid: 1fcb32cd-511e-4ec2-bd44-bb3befa2adf3 kind: Event lastTimestamp: "2026-05-26T16:16:05Z" message: InferenceGraph [switch-graph-c73f1] is Ready metadata: creationTimestamp: "2026-05-26T16:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:16:05Z" name: switch-graph-c73f1.18b329bc27634ba3 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 9dbb39f7-4743-4267-82c6-40005327e713 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T16:17:16.236433Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-df0d2-58df6d94d7-fwb92 namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: da714d88-e1c9-4f61-86e4-e1f154e97f88 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-df0d2-58df6d94d7-fwb92 to ip-10-0-129-89.ec2.internal metadata: creationTimestamp: "2026-05-26T16:17:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T16:17:16Z" name: switch-graph-df0d2-58df6d94d7-fwb92.18b329cc8bc43973 namespace: kserve-ci-e2e-test resourceVersion: "16934" uid: e7c39918-9fc3-4bda-9a5c-0dfb41dad69a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6bfc84f594-l22ft source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:16Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-df0d2-58df6d94d7-fwb92 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: da714d88-e1c9-4f61-86e4-e1f154e97f88 kind: Event lastTimestamp: "2026-05-26T16:17:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-df0d2-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T16:17:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:16Z" name: switch-graph-df0d2-58df6d94d7-fwb92.18b329cc97f060a1 namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 6f3cb908-aa70-4ac9-b9f2-b1d96cc26160 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:17Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-df0d2-58df6d94d7-fwb92 namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: da714d88-e1c9-4f61-86e4-e1f154e97f88 kind: Event lastTimestamp: "2026-05-26T16:17:17Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T16:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T16:17:17Z" name: switch-graph-df0d2-58df6d94d7-fwb92.18b329ccc91b6828 namespace: kserve-ci-e2e-test resourceVersion: "16955" uid: 2157fb17-e979-4872-80f3-04456058104b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df0d2} kind: Pod name: switch-graph-df0d2-58df6d94d7-fwb92 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: da714d88-e1c9-4f61-86e4-e1f154e97f88 kind: Event lastTimestamp: "2026-05-26T16:17:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:82e40282a2f2a4472a45c41fa18200d967bc642df3ede2a9ddc4e0694f6a1107" already present on machine metadata: creationTimestamp: "2026-05-26T16:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:17Z" name: switch-graph-df0d2-58df6d94d7-fwb92.18b329cccac5a241 namespace: kserve-ci-e2e-test resourceVersion: "16957" uid: 30ad3b17-f079-4524-8841-03b542be651d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df0d2} kind: Pod name: switch-graph-df0d2-58df6d94d7-fwb92 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: da714d88-e1c9-4f61-86e4-e1f154e97f88 kind: Event lastTimestamp: "2026-05-26T16:17:17Z" message: 'Created container: switch-graph-df0d2' metadata: creationTimestamp: "2026-05-26T16:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:17Z" name: switch-graph-df0d2-58df6d94d7-fwb92.18b329ccce17c26b namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: daed0797-bff3-47f1-a1db-a497fd0f73ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df0d2} kind: Pod name: switch-graph-df0d2-58df6d94d7-fwb92 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: da714d88-e1c9-4f61-86e4-e1f154e97f88 kind: Event lastTimestamp: "2026-05-26T16:17:17Z" message: Started container switch-graph-df0d2 metadata: creationTimestamp: "2026-05-26T16:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:17:17Z" name: switch-graph-df0d2-58df6d94d7-fwb92.18b329ccce89bb95 namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: 79ddc134-5490-4c34-b1b4-3ec768af43fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:25:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df0d2} kind: Pod name: switch-graph-df0d2-58df6d94d7-fwb92 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: da714d88-e1c9-4f61-86e4-e1f154e97f88 kind: Event lastTimestamp: "2026-05-26T16:25:30Z" message: Stopping container switch-graph-df0d2 metadata: creationTimestamp: "2026-05-26T16:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:30Z" name: switch-graph-df0d2-58df6d94d7-fwb92.18b32a3fb93f2f03 namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: b8b2f7b9-861e-40ee-afd8-2e9300b701b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T16:25:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df0d2} kind: Pod name: switch-graph-df0d2-58df6d94d7-fwb92 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: da714d88-e1c9-4f61-86e4-e1f154e97f88 kind: Event lastTimestamp: "2026-05-26T16:25:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T16:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T16:25:58Z" name: switch-graph-df0d2-58df6d94d7-fwb92.18b32a4051118c54 namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 180c3292-e111-4b86-ad41-675cbf305e37 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-89.ec2.internal source: component: kubelet host: ip-10-0-129-89.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-df0d2-58df6d94d7 namespace: kserve-ci-e2e-test resourceVersion: "16923" uid: af751ae0-4d9b-4acc-8b74-80f2de81b42a kind: Event lastTimestamp: "2026-05-26T16:17:16Z" message: 'Created pod: switch-graph-df0d2-58df6d94d7-fwb92' metadata: creationTimestamp: "2026-05-26T16:17:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:16Z" name: switch-graph-df0d2-58df6d94d7.18b329cc8b3cfcd4 namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: c7d71982-bd85-46aa-8358-b77827671cb3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16922" uid: 28947499-9210-4df5-afcb-b83cdc74d349 kind: Event lastTimestamp: "2026-05-26T16:17:16Z" message: Scaled up replica set switch-graph-df0d2-58df6d94d7 from 0 to 1 metadata: creationTimestamp: "2026-05-26T16:17:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:16Z" name: switch-graph-df0d2.18b329cc8a916789 namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: f8140a14-0cb7-4308-8d42-342ab99b3aed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T16:17:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: 1342b470-1d60-446f-9cca-1b5db6da9a8c kind: Event lastTimestamp: "2026-05-26T16:17:16Z" message: InferenceGraph [switch-graph-df0d2] is Ready metadata: creationTimestamp: "2026-05-26T16:17:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T16:17:16Z" name: switch-graph-df0d2.18b329cc8e4a5373 namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: a876f290-cab8-485d-b301-da82685b6646 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:17:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 9f4962da-89eb-44c0-8d49-02c97b352214 kind: Event lastTimestamp: "2026-05-26T16:17:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:46Z" name: switch-graph-df0d2.18b329d009252a43 namespace: kserve-ci-e2e-test resourceVersion: "17203" uid: 88e1509f-f09f-495d-aec8-c7935390b971 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T16:17:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-df0d2 namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 9f4962da-89eb-44c0-8d49-02c97b352214 kind: Event lastTimestamp: "2026-05-26T16:17:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T16:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T16:17:46Z" name: switch-graph-df0d2.18b329d0098a80bd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: c79ae498-e4e1-4fe7-a0c0-e47ca91a3b80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34439"