--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:39:16.244310Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-43ae3-65f97db95d-lv2vz namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: a6f2fa2f-b96b-4fd2-b630-a8ef0a4e0222 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-43ae3-65f97db95d-lv2vz to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:39:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:39:16Z" name: ensemble-graph-43ae3-65f97db95d-lv2vz.18be3b2a8232b3b3 namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: 588ac602-8c12-4200-b13c-44d1354b52c2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:16Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-43ae3-65f97db95d-lv2vz namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: a6f2fa2f-b96b-4fd2-b630-a8ef0a4e0222 kind: Event lastTimestamp: "2026-07-01T17:39:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-43ae3-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:16Z" name: ensemble-graph-43ae3-65f97db95d-lv2vz.18be3b2a8fad9c55 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: e1c8bc01-b014-4acf-8b5e-7b7d6c8c77b0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:17Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-43ae3-65f97db95d-lv2vz namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: a6f2fa2f-b96b-4fd2-b630-a8ef0a4e0222 kind: Event lastTimestamp: "2026-07-01T17:39:17Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:39:17Z" name: ensemble-graph-43ae3-65f97db95d-lv2vz.18be3b2ac02c9a8d namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 5634d7ed-dbb1-4119-9c00-d215d43e1e7c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-43ae3} kind: Pod name: ensemble-graph-43ae3-65f97db95d-lv2vz namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: a6f2fa2f-b96b-4fd2-b630-a8ef0a4e0222 kind: Event lastTimestamp: "2026-07-01T17:39:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:17Z" name: ensemble-graph-43ae3-65f97db95d-lv2vz.18be3b2ac1cdaf50 namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: 07a833bb-f524-4edf-9ee1-74ad01b176cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-43ae3} kind: Pod name: ensemble-graph-43ae3-65f97db95d-lv2vz namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: a6f2fa2f-b96b-4fd2-b630-a8ef0a4e0222 kind: Event lastTimestamp: "2026-07-01T17:39:17Z" message: 'Created container: ensemble-graph-43ae3' metadata: creationTimestamp: "2026-07-01T17:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:17Z" name: ensemble-graph-43ae3-65f97db95d-lv2vz.18be3b2ac466d675 namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: 67255118-e6e6-4c6a-92fd-806915a1643c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-43ae3} kind: Pod name: ensemble-graph-43ae3-65f97db95d-lv2vz namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: a6f2fa2f-b96b-4fd2-b630-a8ef0a4e0222 kind: Event lastTimestamp: "2026-07-01T17:39:17Z" message: Started container ensemble-graph-43ae3 metadata: creationTimestamp: "2026-07-01T17:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:17Z" name: ensemble-graph-43ae3-65f97db95d-lv2vz.18be3b2ac4dd3197 namespace: kserve-ci-e2e-test resourceVersion: "21271" uid: 756f885c-884c-4070-98f4-30a73a5380b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-43ae3} kind: Pod name: ensemble-graph-43ae3-65f97db95d-lv2vz namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: a6f2fa2f-b96b-4fd2-b630-a8ef0a4e0222 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Stopping container ensemble-graph-43ae3 metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:26Z" name: ensemble-graph-43ae3-65f97db95d-lv2vz.18be3b2cd9a1ca6a namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: a285325c-6445-4498-92ec-bfa50838c83c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:39:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-43ae3} kind: Pod name: ensemble-graph-43ae3-65f97db95d-lv2vz namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: a6f2fa2f-b96b-4fd2-b630-a8ef0a4e0222 kind: Event lastTimestamp: "2026-07-01T17:39:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:53Z" name: ensemble-graph-43ae3-65f97db95d-lv2vz.18be3b2d5e0bfd22 namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: 82ca234a-10e3-4e28-b7dd-b22cdc7cbae0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-43ae3-65f97db95d namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: b277b201-8517-45f2-98b5-4dce2dd689f5 kind: Event lastTimestamp: "2026-07-01T17:39:16Z" message: 'Created pod: ensemble-graph-43ae3-65f97db95d-lv2vz' metadata: creationTimestamp: "2026-07-01T17:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:16Z" name: ensemble-graph-43ae3-65f97db95d.18be3b2a81c8ec29 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 3dcdb01b-6c23-4bf3-b79a-b81d8c709060 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-43ae3 namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: 55fd270a-ce1a-4162-a208-aa7bad2611fc kind: Event lastTimestamp: "2026-07-01T17:39:16Z" message: Scaled up replica set ensemble-graph-43ae3-65f97db95d from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:16Z" name: ensemble-graph-43ae3.18be3b2a80dad405 namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: 5fff70de-9627-4ea1-be0e-987ec80ef573 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-43ae3 namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: cf2e6df6-44f7-42c6-90a0-05f23d5b6ba0 kind: Event lastTimestamp: "2026-07-01T17:39:16Z" message: InferenceGraph [ensemble-graph-43ae3] is Ready metadata: creationTimestamp: "2026-07-01T17:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:16Z" name: ensemble-graph-43ae3.18be3b2a882cf5c0 namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: e9c67219-8260-4414-a684-64d7456e2aad reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:40:26.522982Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c4578-749655d7b7-hd9bg namespace: kserve-ci-e2e-test resourceVersion: "22198" uid: 4de8cd46-61b5-43cd-a3bb-dd661796db92 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c4578-749655d7b7-hd9bg to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:40:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:40:26Z" name: ensemble-graph-c4578-749655d7b7-hd9bg.18be3b3adf220cd1 namespace: kserve-ci-e2e-test resourceVersion: "22203" uid: e3e5f637-ffb8-47cc-baf9-2483080816f5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c4578-749655d7b7-hd9bg namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4de8cd46-61b5-43cd-a3bb-dd661796db92 kind: Event lastTimestamp: "2026-07-01T17:40:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-c4578-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:26Z" name: ensemble-graph-c4578-749655d7b7-hd9bg.18be3b3ae9b0e8bb namespace: kserve-ci-e2e-test resourceVersion: "22216" uid: 9278855d-65c8-4713-ac6b-7f8e7b78f1a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:27Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c4578-749655d7b7-hd9bg namespace: kserve-ci-e2e-test resourceVersion: "22207" uid: 4de8cd46-61b5-43cd-a3bb-dd661796db92 kind: Event lastTimestamp: "2026-07-01T17:40:27Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:40:27Z" name: ensemble-graph-c4578-749655d7b7-hd9bg.18be3b3b1ce668b5 namespace: kserve-ci-e2e-test resourceVersion: "22233" uid: 11c24129-6959-42a6-aafe-04e768b9573f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c4578} kind: Pod name: ensemble-graph-c4578-749655d7b7-hd9bg namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4de8cd46-61b5-43cd-a3bb-dd661796db92 kind: Event lastTimestamp: "2026-07-01T17:40:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:27Z" name: ensemble-graph-c4578-749655d7b7-hd9bg.18be3b3b1e32c77c namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: 365fd43c-91e3-4be2-a957-7f2d3756e19c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c4578} kind: Pod name: ensemble-graph-c4578-749655d7b7-hd9bg namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4de8cd46-61b5-43cd-a3bb-dd661796db92 kind: Event lastTimestamp: "2026-07-01T17:40:27Z" message: 'Created container: ensemble-graph-c4578' metadata: creationTimestamp: "2026-07-01T17:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:27Z" name: ensemble-graph-c4578-749655d7b7-hd9bg.18be3b3b214d545f namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 8eb8f21f-294b-467a-b33d-1d50d78d4dc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c4578} kind: Pod name: ensemble-graph-c4578-749655d7b7-hd9bg namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4de8cd46-61b5-43cd-a3bb-dd661796db92 kind: Event lastTimestamp: "2026-07-01T17:40:27Z" message: Started container ensemble-graph-c4578 metadata: creationTimestamp: "2026-07-01T17:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:27Z" name: ensemble-graph-c4578-749655d7b7-hd9bg.18be3b3b21c98d28 namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 62ff73cf-eeef-4d24-b7fd-30d9828de775 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c4578} kind: Pod name: ensemble-graph-c4578-749655d7b7-hd9bg namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4de8cd46-61b5-43cd-a3bb-dd661796db92 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Stopping container ensemble-graph-c4578 metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: ensemble-graph-c4578-749655d7b7-hd9bg.18be3bae0e84045a namespace: kserve-ci-e2e-test resourceVersion: "26018" uid: 842d64e0-175f-4021-9b2a-90581bbeb658 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:48:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c4578} kind: Pod name: ensemble-graph-c4578-749655d7b7-hd9bg namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4de8cd46-61b5-43cd-a3bb-dd661796db92 kind: Event lastTimestamp: "2026-07-01T17:49:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:08Z" name: ensemble-graph-c4578-749655d7b7-hd9bg.18be3baea8432bea namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: 1d483a81-ade2-4f5b-931f-407938270390 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c4578-749655d7b7 namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: 92e62b53-19d2-4508-9f47-4d3244197bc4 kind: Event lastTimestamp: "2026-07-01T17:40:26Z" message: 'Created pod: ensemble-graph-c4578-749655d7b7-hd9bg' metadata: creationTimestamp: "2026-07-01T17:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:26Z" name: ensemble-graph-c4578-749655d7b7.18be3b3ade9a7958 namespace: kserve-ci-e2e-test resourceVersion: "22199" uid: 4a55b53b-5057-448a-ad07-ac712c1e6c87 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c4578 namespace: kserve-ci-e2e-test resourceVersion: "22192" uid: 04ee3e36-222a-4f3a-a20f-76544620e76b kind: Event lastTimestamp: "2026-07-01T17:40:26Z" message: Scaled up replica set ensemble-graph-c4578-749655d7b7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:26Z" name: ensemble-graph-c4578.18be3b3addbca2cc namespace: kserve-ci-e2e-test resourceVersion: "22196" uid: 0538f7d3-a4b8-4a71-8f04-3a3b0804726f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c4578 namespace: kserve-ci-e2e-test resourceVersion: "22211" uid: 776a4649-03d0-41ce-92f8-caaebc7efe11 kind: Event lastTimestamp: "2026-07-01T17:40:26Z" message: InferenceGraph [ensemble-graph-c4578] is Ready metadata: creationTimestamp: "2026-07-01T17:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:26Z" name: ensemble-graph-c4578.18be3b3ae58cf739 namespace: kserve-ci-e2e-test resourceVersion: "22213" uid: 69ce117a-8409-4476-bdbe-4467dc24d6a4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c4578 namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: 776a4649-03d0-41ce-92f8-caaebc7efe11 kind: Event lastTimestamp: "2026-07-01T17:40:26Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-c4578": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-c4578": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:26Z" name: ensemble-graph-c4578.18be3b3ae9040a43 namespace: kserve-ci-e2e-test resourceVersion: "22214" uid: 385c1d29-b5c9-4f88-b5eb-8148b13f472a reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c4578 namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: 776a4649-03d0-41ce-92f8-caaebc7efe11 kind: Event lastTimestamp: "2026-07-01T17:40:26Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-c4578": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:26Z" name: ensemble-graph-c4578.18be3b3ae9046cf5 namespace: kserve-ci-e2e-test resourceVersion: "22215" uid: 1c993e0f-871a-4a21-a966-08fbb179bd99 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:40:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c4578 namespace: kserve-ci-e2e-test resourceVersion: "22193" uid: 536d788a-a435-41d5-8c27-433dde2f7930 kind: Event lastTimestamp: "2026-07-01T17:41:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:11Z" name: ensemble-graph-c4578.18be3b3e5c2be063 namespace: kserve-ci-e2e-test resourceVersion: "22601" uid: 7baace12-696c-482f-8c50-0a8232e76025 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:40:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c4578 namespace: kserve-ci-e2e-test resourceVersion: "22193" uid: 536d788a-a435-41d5-8c27-433dde2f7930 kind: Event lastTimestamp: "2026-07-01T17:41:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:11Z" name: ensemble-graph-c4578.18be3b3e5c8846ac namespace: kserve-ci-e2e-test resourceVersion: "22602" uid: 54320eaa-0329-45d1-a381-bbb17cf3e242 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:48:41.508567Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-10517-predictor-f6447f778-924c9 to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:48:41Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bae1e92c18a namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 19cfc1b6-704e-4977-a438-e754c9347abe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:48:41Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bae38282834 namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: c9eb1884-6c5b-4fd7-96a4-ebaf2eec042f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bae396970af namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 63ebfbad-450f-4f26-94c6-64480df08aae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:48:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:42Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bae3c7ada65 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: b9cf5c03-97dc-4f1c-abc2-79a00b5c2dad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:48:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:42Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bae3cf36107 namespace: kserve-ci-e2e-test resourceVersion: "26134" uid: d8629e2f-c932-4e6f-a048-09fa90d57e55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:48:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:42Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bae3cfdc0c5 namespace: kserve-ci-e2e-test resourceVersion: "26136" uid: 0e570b2e-5057-4850-a8ad-7913c0cac9d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:48:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:42Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bae3fe60438 namespace: kserve-ci-e2e-test resourceVersion: "26139" uid: 15f67225-0bac-43ba-a089-dbd7abe05529 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:48:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:42Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bae405bffca namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: 9df5db67-cf3c-4b39-b11b-d58410853542 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:48:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:49:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:18Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bae6a029e17 namespace: kserve-ci-e2e-test resourceVersion: "26446" uid: dc536707-ae70-4adc-bde8-1433aedc35ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:51Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bbe78e6d8d8 namespace: kserve-ci-e2e-test resourceVersion: "26983" uid: 2d45133d-e905-4d27-bf02-ec8b4c28cea9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:51Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bbe78e78eb2 namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: 0d2375ab-581c-47e1-a334-1a1c9ea0708b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-10517-predictor-f6447f778-924c9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 21db6b05-cc24-497d-83d1-595ccc04fc7b kind: Event lastTimestamp: "2026-07-01T17:49:53Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:53Z" name: error-404-isvc-10517-predictor-f6447f778-924c9.18be3bbef2274127 namespace: kserve-ci-e2e-test resourceVersion: "27053" uid: e3f0f327-2550-49eb-be84-a8fce55ddd24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-10517-predictor-f6447f778 namespace: kserve-ci-e2e-test resourceVersion: "26075" uid: 1a5691e9-effa-4960-8a7e-35f7ebdda63c kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: 'Created pod: error-404-isvc-10517-predictor-f6447f778-924c9' metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:48:41Z" name: error-404-isvc-10517-predictor-f6447f778.18be3bae1e272a79 namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: ed5309f5-606c-46e2-a5d4-4d404cc71abc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: e9cdf6ea-f23d-46e3-8e95-7355aa3ce342 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Scaled up replica set error-404-isvc-10517-predictor-f6447f778 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:48:41Z" name: error-404-isvc-10517-predictor.18be3bae1c92fae6 namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 6b20b747-4714-4fa3-97da-02febac8761c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: 3d0965b0-1a0a-40ea-9433-e5638e8bf965 kind: Event lastTimestamp: "2026-07-01T17:49:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:26Z" name: error-404-isvc-10517-predictor.18be3bb19b0ee0d3 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 512331ab-4797-4c67-8f91-aa7142222db9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: 3d0965b0-1a0a-40ea-9433-e5638e8bf965 kind: Event lastTimestamp: "2026-07-01T17:49:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:26Z" name: error-404-isvc-10517-predictor.18be3bb19b781f4a namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: d9ebbd71-2588-4d05-87c3-fe5bba9c4223 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:49:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: 3d0965b0-1a0a-40ea-9433-e5638e8bf965 kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:41Z" name: error-404-isvc-10517-predictor.18be3bb51a24161a namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: 36b76f42-74ed-408a-8dfe-756f1ec0f519 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:49:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: 3d0965b0-1a0a-40ea-9433-e5638e8bf965 kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:41Z" name: error-404-isvc-10517-predictor.18be3bb51a8b4585 namespace: kserve-ci-e2e-test resourceVersion: "26801" uid: cb45d546-8a44-4322-99cd-73a36a7acaf8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-10517 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 2f3bf6cc-804d-4ae8-afcd-1fc758676364 kind: Event lastTimestamp: "2026-07-01T17:49:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:28Z" name: error-404-isvc-10517.18be3bae1c979216 namespace: kserve-ci-e2e-test resourceVersion: "26525" uid: 88d80860-7473-42be-aa2b-d5867f22f7df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-10517 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 2f3bf6cc-804d-4ae8-afcd-1fc758676364 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-10517": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-10517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:48:41Z" name: error-404-isvc-10517.18be3bae1d2ce310 namespace: kserve-ci-e2e-test resourceVersion: "26080" uid: bad99ce9-dcd6-49c0-a992-8ed9ed8d2c40 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-10517 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 2f3bf6cc-804d-4ae8-afcd-1fc758676364 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-10517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:48:41Z" name: error-404-isvc-10517.18be3bae1d2d3366 namespace: kserve-ci-e2e-test resourceVersion: "26083" uid: b79794b6-f96f-4bf9-9270-7c5a44ec50a5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-10517 namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: 2f3bf6cc-804d-4ae8-afcd-1fc758676364 kind: Event lastTimestamp: "2026-07-01T17:49:28Z" message: InferenceService [error-404-isvc-10517] is Ready metadata: creationTimestamp: "2026-07-01T17:49:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:28Z" name: error-404-isvc-10517.18be3bb925324098 namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: 301a493b-1e4a-43a2-9b96-e64b1a153c5c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:49:51.781728Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26992" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-42b28-predictor-544cbc8654-8458m to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:49:51Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3bbe7b2e1caf namespace: kserve-ci-e2e-test resourceVersion: "26996" uid: fbad36b8-7902-4ece-b977-8eb25288a210 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "27002" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:49:52Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3bbe95145e27 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: 300d41fc-1837-4b34-8a12-ab8c9677382d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3bbe96747a72 namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 540d1bec-967b-4859-bf6a-cfdcdf56ce4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3bbe9957eea8 namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: 834625fd-b164-4476-af98-b7ede6acb26f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3bbe99ce4af0 namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: 7b934c6b-fbf1-4437-844c-65103c20e40c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3bbe99d98646 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: de5acac6-f05a-4b7c-bfc5-eb683daec5f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3bbe9d912a75 namespace: kserve-ci-e2e-test resourceVersion: "27036" uid: 21074776-e538-414d-b34b-7bcc31e5d8c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3bbe9e0a626a namespace: kserve-ci-e2e-test resourceVersion: "27037" uid: 9ab5a105-6ff1-441a-8950-6d5c6639df17 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:49:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:59:10Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:10Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3bbf04325143 namespace: kserve-ci-e2e-test resourceVersion: "31147" uid: 84689390-58ed-4e03-aaf3-5bb5318b5b64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:59:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:06Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3c3f9d4a392e namespace: kserve-ci-e2e-test resourceVersion: "31110" uid: 6fc62805-604f-4b17-8f7f-2fd64817fa91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:59:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:06Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3c3f9d4bcd29 namespace: kserve-ci-e2e-test resourceVersion: "31111" uid: f4648f88-396c-46c7-aa82-cc7a53e46785 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42b28-predictor-544cbc8654-8458m namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 8c0e28fa-dd0b-455f-8be4-d9bb36f8833b kind: Event lastTimestamp: "2026-07-01T17:59:10Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:10Z" name: error-404-isvc-42b28-predictor-544cbc8654-8458m.18be3c4078bb0d06 namespace: kserve-ci-e2e-test resourceVersion: "31146" uid: e320c9dc-3cbc-4a17-baae-8515d8f0b147 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-42b28-predictor-544cbc8654 namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: 47f53fca-17a8-45f0-853c-e264de2fad1c kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: 'Created pod: error-404-isvc-42b28-predictor-544cbc8654-8458m' metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:51Z" name: error-404-isvc-42b28-predictor-544cbc8654.18be3bbe7ac68353 namespace: kserve-ci-e2e-test resourceVersion: "26993" uid: 00bf18f0-7592-44b0-bfa2-eca27eeb35fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "26979" uid: b0244983-37e3-489f-aa5d-9d8cb01d6717 kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: Scaled up replica set error-404-isvc-42b28-predictor-544cbc8654 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:51Z" name: error-404-isvc-42b28-predictor.18be3bbe7913737b namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: 84f778dc-d25f-4f29-b61f-6f3676c40958 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "26981" uid: 2f681425-b255-4874-8d3e-62c23b496327 kind: Event lastTimestamp: "2026-07-01T17:50:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:21Z" name: error-404-isvc-42b28-predictor.18be3bc1f76d262a namespace: kserve-ci-e2e-test resourceVersion: "27293" uid: d28a57d8-0cb5-4bf5-bce7-b42ccaccb5dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "26981" uid: 2f681425-b255-4874-8d3e-62c23b496327 kind: Event lastTimestamp: "2026-07-01T17:50:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:21Z" name: error-404-isvc-42b28-predictor.18be3bc1f7c9f845 namespace: kserve-ci-e2e-test resourceVersion: "27294" uid: 75593181-cb6d-49b5-ad5e-d6992393da56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "27169" uid: 2f681425-b255-4874-8d3e-62c23b496327 kind: Event lastTimestamp: "2026-07-01T17:51:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:06Z" name: error-404-isvc-42b28-predictor.18be3bc8f56320ab namespace: kserve-ci-e2e-test resourceVersion: "27758" uid: 01f25eca-6f49-4a8f-97c9-086f88490951 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "27169" uid: 2f681425-b255-4874-8d3e-62c23b496327 kind: Event lastTimestamp: "2026-07-01T17:51:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:06Z" name: error-404-isvc-42b28-predictor.18be3bc8f5c784a3 namespace: kserve-ci-e2e-test resourceVersion: "27759" uid: 0905272e-6688-4796-8581-3d718c95d925 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42b28 namespace: kserve-ci-e2e-test resourceVersion: "26954" uid: d9d4e6d2-5ada-49fc-a630-4d49f4e4afa1 kind: Event lastTimestamp: "2026-07-01T17:50:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:40Z" name: error-404-isvc-42b28.18be3bbe791ae036 namespace: kserve-ci-e2e-test resourceVersion: "27520" uid: 8b3dfc6c-bafc-41b9-8ee1-7c93b52de537 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42b28 namespace: kserve-ci-e2e-test resourceVersion: "26954" uid: d9d4e6d2-5ada-49fc-a630-4d49f4e4afa1 kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-42b28": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-42b28": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:51Z" name: error-404-isvc-42b28.18be3bbe799849f3 namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: 6612b32f-d989-4e6b-b4ce-a6bf3830a1e4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42b28 namespace: kserve-ci-e2e-test resourceVersion: "26954" uid: d9d4e6d2-5ada-49fc-a630-4d49f4e4afa1 kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-42b28": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:51Z" name: error-404-isvc-42b28.18be3bbe799899c1 namespace: kserve-ci-e2e-test resourceVersion: "26990" uid: d9116179-a144-42c9-9157-586b1a550911 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42b28 namespace: kserve-ci-e2e-test resourceVersion: "27517" uid: d9d4e6d2-5ada-49fc-a630-4d49f4e4afa1 kind: Event lastTimestamp: "2026-07-01T17:50:40Z" message: InferenceService [error-404-isvc-42b28] is Ready metadata: creationTimestamp: "2026-07-01T17:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:40Z" name: error-404-isvc-42b28.18be3bc9bf9addc9 namespace: kserve-ci-e2e-test resourceVersion: "27518" uid: 70bd88a3-60e1-41f4-9e24-fa9511bfb5b7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:38:16.428900Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-43ae3-predictor-669cb6b886-wfs2q to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:38:16Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1c94ebe308 namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 0d2a21d7-ba9c-4a5d-b98a-97c1b95f8d1d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-43ae3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:16Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1c9fff93ce namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 2dab5fe1-1fd6-4440-b5dd-ad84192b0120 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:38:17Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1cd223098d namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: e8e99e8e-572b-4481-8146-5df494fc2556 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1cd38213ce namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 1af2d96b-f700-4637-a169-3c3de5abf129 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1cd6bbe5c6 namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: e4835172-23ae-4fe1-bf14-b01bb3859ecc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1cd71e7677 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 80ed5d7a-0f14-4702-85c8-abd6194d79bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1cd729862c namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: 1a7e32bd-92ba-4c5c-a532-6efe59349211 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1cd9d28a67 namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: 25d153ac-9e7f-4c6e-926b-0f56c0ac882e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1cda62238a namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: a5b54f7b-a034-44a5-8cb0-e8bb3b0b2a3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:38:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:55Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b1d3e611115 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: b20f4b77-c1db-40ce-8742-0e2c8a567200 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:26Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b2ce9e32a6c namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: e088b35f-ea80-489b-9848-1dc0b08e74ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: f8e53a5f-7554-4eff-a0d1-b3ef8bf10b67 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:26Z" name: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q.18be3b2ce9e45123 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: de582128-8842-4501-9983-ee0201e8b0cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-43ae3-predictor-669cb6b886 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 50a7268f-2b19-4122-9033-8108e14eb9ef kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: 'Created pod: error-404-isvc-43ae3-predictor-669cb6b886-wfs2q' metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:38:16Z" name: error-404-isvc-43ae3-predictor-669cb6b886.18be3b1c946d837b namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 9f9de441-fad0-4f34-8062-b345c2da291a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: f70bcb8c-a5fd-4945-a332-86e7700e8b35 kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: Scaled up replica set error-404-isvc-43ae3-predictor-669cb6b886 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:38:16Z" name: error-404-isvc-43ae3-predictor.18be3b1c92bb2204 namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 43862e1e-5f2e-4640-a66e-d3503a2732dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:38:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 94b55f00-7387-4310-9891-d665f6a0613b kind: Event lastTimestamp: "2026-07-01T17:38:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:38:46Z" name: error-404-isvc-43ae3-predictor.18be3b201155959d namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 223bf7ed-d48a-4e47-9696-b8340a3625da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:38:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 94b55f00-7387-4310-9891-d665f6a0613b kind: Event lastTimestamp: "2026-07-01T17:38:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:38:46Z" name: error-404-isvc-43ae3-predictor.18be3b2011be3a20 namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 24058cd7-adc0-413f-a87d-b3c5083f2989 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 94b55f00-7387-4310-9891-d665f6a0613b kind: Event lastTimestamp: "2026-07-01T17:39:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:16Z" name: error-404-isvc-43ae3-predictor.18be3b270f2fb452 namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: 1b0ab768-d848-414e-ac43-932b3bfdcbc8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 94b55f00-7387-4310-9891-d665f6a0613b kind: Event lastTimestamp: "2026-07-01T17:39:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:16Z" name: error-404-isvc-43ae3-predictor.18be3b270faeb2a9 namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 5632e343-036d-4729-ae35-5b8bee185683 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43ae3 namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 4dea78e7-24c9-40be-aa54-212ab6bf8995 kind: Event lastTimestamp: "2026-07-01T17:39:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:05Z" name: error-404-isvc-43ae3.18be3b1c92e0f264 namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: c8867761-ffb0-4bc3-89e8-b4c1b3167411 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43ae3 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: 4dea78e7-24c9-40be-aa54-212ab6bf8995 kind: Event lastTimestamp: "2026-07-01T17:39:05Z" message: InferenceService [error-404-isvc-43ae3] is Ready metadata: creationTimestamp: "2026-07-01T17:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:05Z" name: error-404-isvc-43ae3.18be3b27fe1d3ae7 namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: b7b25833-18e1-4a62-adbc-ff82be410598 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:49:37.210197Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26684" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:49:37Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbb16a62d2b namespace: kserve-ci-e2e-test resourceVersion: "26688" uid: a1b82370-78e4-4b8a-9406-71f184bb4bdc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4822a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbb231e56b7 namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: 094ce518-fe4c-4b24-bb3c-67f83f7cff9f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T17:49:38Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:49:38Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbb54187660 namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: 06044bc5-c4be-405c-9aea-1ff484a88976 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T17:49:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:38Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbb55706062 namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: a779f5e4-e2a6-4894-a7a7-b83e9a9c9ff3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T17:49:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:38Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbb58337e0e namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: 45596cff-b796-425a-90d5-9c7fab52e100 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T17:49:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:38Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbb58b08056 namespace: kserve-ci-e2e-test resourceVersion: "26728" uid: 227e8d1c-987d-450b-8adc-3bfbf1a7af08 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T17:49:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:38Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbb58bbb9a8 namespace: kserve-ci-e2e-test resourceVersion: "26729" uid: fa20cab7-9535-44fa-b4ef-8a613121b2f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T17:49:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:38Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbb5bde5386 namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: 347f492b-e90f-4bfb-a641-89a6ca69cc04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T17:49:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:38Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbb5c637ef5 namespace: kserve-ci-e2e-test resourceVersion: "26731" uid: b3e2d078-f961-4210-aada-f6be5e0a736b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T17:50:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:16Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3bbbbd80002f namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: 76ad5283-ee2c-4a77-8ad9-959287832a47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T18:06:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:06:56Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3cad1a402d57 namespace: kserve-ci-e2e-test resourceVersion: "34379" uid: 3e8e4aa1-c266-4889-915f-5da13e75a92f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: 86b90960-cd85-4aa7-b01e-100c95520755 kind: Event lastTimestamp: "2026-07-01T18:06:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:06:56Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56.18be3cad1a415f71 namespace: kserve-ci-e2e-test resourceVersion: "34380" uid: b480c365-05f4-45c4-afbf-96d70942bd23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4822a-predictor-6fb5b8b67b namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: eb8788f9-fef2-4cd6-8870-f5668ad2c089 kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: 'Created pod: error-404-isvc-4822a-predictor-6fb5b8b67b-jnl56' metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:37Z" name: error-404-isvc-4822a-predictor-6fb5b8b67b.18be3bbb163b4878 namespace: kserve-ci-e2e-test resourceVersion: "26685" uid: eeac1562-be84-495f-810a-70a2dee57187 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 171083f3-be66-4b98-845f-16e207851283 kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Scaled up replica set error-404-isvc-4822a-predictor-6fb5b8b67b from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:37Z" name: error-404-isvc-4822a-predictor.18be3bbb1491199c namespace: kserve-ci-e2e-test resourceVersion: "26676" uid: 22adf4c0-5650-4cf1-93f0-025b073a04b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: 06281a28-9e18-4200-a08c-4696733bd921 kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:52Z" name: error-404-isvc-4822a-predictor.18be3bbe93168dd2 namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: 6cbfade2-8190-4be3-8c03-8cb1cb3d88b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: 06281a28-9e18-4200-a08c-4696733bd921 kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:52Z" name: error-404-isvc-4822a-predictor.18be3bbe937c7773 namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: b51c7949-72da-48d4-8aef-ee4cb9011f05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:50:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 06281a28-9e18-4200-a08c-4696733bd921 kind: Event lastTimestamp: "2026-07-01T17:51:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:07Z" name: error-404-isvc-4822a-predictor.18be3bc21219e394 namespace: kserve-ci-e2e-test resourceVersion: "27768" uid: e0730eb3-054e-48cf-9951-b16520eda753 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:50:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 06281a28-9e18-4200-a08c-4696733bd921 kind: Event lastTimestamp: "2026-07-01T17:51:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:07Z" name: error-404-isvc-4822a-predictor.18be3bc212771546 namespace: kserve-ci-e2e-test resourceVersion: "27769" uid: 833b4a7c-8f6d-46e6-a77c-f7bc967d49b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4822a namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: 80ae9f00-5721-4a26-afe7-0ea061ddee42 kind: Event lastTimestamp: "2026-07-01T17:50:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:26Z" name: error-404-isvc-4822a.18be3bbb149dda0a namespace: kserve-ci-e2e-test resourceVersion: "27349" uid: fcb7607d-febf-4a26-ab23-e9a0ae01e49f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4822a namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: 80ae9f00-5721-4a26-afe7-0ea061ddee42 kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-4822a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4822a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:37Z" name: error-404-isvc-4822a.18be3bbb154e8664 namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: c33bac38-6190-43fe-901c-a43b964b2507 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4822a namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: 80ae9f00-5721-4a26-afe7-0ea061ddee42 kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4822a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:37Z" name: error-404-isvc-4822a.18be3bbb154efea1 namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: 79514cc7-2dc5-472f-9012-85a409832e1d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4822a namespace: kserve-ci-e2e-test resourceVersion: "27345" uid: 80ae9f00-5721-4a26-afe7-0ea061ddee42 kind: Event lastTimestamp: "2026-07-01T17:50:26Z" message: InferenceService [error-404-isvc-4822a] is Ready metadata: creationTimestamp: "2026-07-01T17:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:26Z" name: error-404-isvc-4822a.18be3bc679de17d7 namespace: kserve-ci-e2e-test resourceVersion: "27346" uid: 2f1e6b99-7936-41f6-a44c-fa4e9bb076d7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:29:01.279311Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3a9b536d7b78 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: f9cb7d5f-030b-4cf6-912c-3803dd27573e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3a9b6ce07de9 namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: fc32b570-48fe-47ca-a106-905ca9464e4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3a9b6e24fe86 namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: 5abc2b30-5366-4f02-8ce3-8999f10bba87 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3a9b71752554 namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: dc06164f-8f6a-4470-9d7d-a2534196d35e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3a9b71de002b namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 2ad60985-86c7-41ee-8d3c-a5ac3226cec8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3a9b71e92363 namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: 0ccded9a-63e3-4bd3-94ec-dd268ab78dba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3a9b745020dd namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 85fc1904-e571-4cad-b623-f013be9089e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3a9b74b5ca40 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 3e4490f0-4514-4986-9b1d-4e94bbc579e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:29:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:38:19Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:19Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3a9bbcaba7b4 namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: e476765b-8d84-4dcc-b721-7cde0984d773 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:16Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3b1c91450aef namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 1b098bfa-d20c-4f09-80f1-f90c9e2fabae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:16Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3b1c9145c606 namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: 1a9ae737-3d6c-4111-9a6b-01dafc7faf28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 51c0df56-15c4-4eaa-988f-1c5b154853af kind: Event lastTimestamp: "2026-07-01T17:38:19Z" message: 'Readiness probe failed: Get "https://10.132.0.20:8643/healthz": dial tcp 10.132.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:19Z" name: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g.18be3b1d3104a20e namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: 9ea64775-64ee-4f4a-a07a-02d205c5f508 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6ac71-predictor-779d48fcd6 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: c63e4815-0e57-4341-a050-f3c79b24ed48 kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'Created pod: error-404-isvc-6ac71-predictor-779d48fcd6-2lz9g' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor-779d48fcd6.18be3a9b53075a18 namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: a625ab33-bab2-49b9-9939-d4a9750da081 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: d60b1098-2f58-47df-bb63-c047b25465ea kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Scaled up replica set error-404-isvc-6ac71-predictor-779d48fcd6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71-predictor.18be3a9b516cd091 namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 4e5d3ea8-6037-4fc2-ab8e-f3454c123c9a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 0525a466-54b2-479b-8d75-6a7b977f536e kind: Event lastTimestamp: "2026-07-01T17:29:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:31Z" name: error-404-isvc-6ac71-predictor.18be3a9ecfedcb57 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: ef92501e-62f7-49cb-a135-cf906eee01f8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 0525a466-54b2-479b-8d75-6a7b977f536e kind: Event lastTimestamp: "2026-07-01T17:29:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:31Z" name: error-404-isvc-6ac71-predictor.18be3a9ed050ee36 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 15fd3cfc-11ff-4fc6-92c2-2641720f91e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 0525a466-54b2-479b-8d75-6a7b977f536e kind: Event lastTimestamp: "2026-07-01T17:30:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:01Z" name: error-404-isvc-6ac71-predictor.18be3aa5cdbe0481 namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: e219e404-4d73-4a18-b06e-fd796a580d53 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 0525a466-54b2-479b-8d75-6a7b977f536e kind: Event lastTimestamp: "2026-07-01T17:30:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:01Z" name: error-404-isvc-6ac71-predictor.18be3aa5ce26a724 namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: d75df8ae-1008-466b-9bd4-aa49132341b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 4d82b772-5751-4852-9ef9-c00fc98bfa0c kind: Event lastTimestamp: "2026-07-01T17:29:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:49Z" name: error-404-isvc-6ac71.18be3a9b51735f52 namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: e48cc974-cd21-4913-840b-fcf6d4e7ba53 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 4d82b772-5751-4852-9ef9-c00fc98bfa0c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-6ac71": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6ac71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71.18be3a9b5202fef0 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: a068f166-4c4b-4deb-bbce-45c9aa2471a3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 4d82b772-5751-4852-9ef9-c00fc98bfa0c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6ac71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-6ac71.18be3a9b5203408c namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 5244602d-20cb-4793-bf7c-308b6af1fb0d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: 4d82b772-5751-4852-9ef9-c00fc98bfa0c kind: Event lastTimestamp: "2026-07-01T17:29:49Z" message: InferenceService [error-404-isvc-6ac71] is Ready metadata: creationTimestamp: "2026-07-01T17:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:49Z" name: error-404-isvc-6ac71.18be3aa677f4adc3 namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: daeb6497-3693-4f45-9b27-3562ddaa7316 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:29:47.291210Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:29:47Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa609f34cd6 namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: aef9f7a1-8276-4bc7-94a9-5100326e6aa4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-766f3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa6145dcbca namespace: kserve-ci-e2e-test resourceVersion: "16759" uid: 4d6b87c2-9868-4a8d-8ea7-ff01f25fd080 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:29:48Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:29:48Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa646f47290 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 2cfe3980-12f4-4661-abf7-61f06de3c3a7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:29:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:48Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa6484d7bd7 namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: 6b84c29b-90c0-4eea-a5c1-d682aba929f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:29:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:48Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa64cbcf6d5 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: ff96bb58-4128-4478-8aa5-3e419d0194d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:29:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:48Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa64d20d45d namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 4305e883-9307-4199-833e-2f33e8968e77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:29:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:48Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa64d2ae1c9 namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: 76437187-f2af-43cb-845c-d140652c9f25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:29:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:48Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa64fb17ae4 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: bcd20807-2003-4305-800b-510dc20e9e7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:29:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:48Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa6501feee6 namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: 8a94a234-9bd8-42fa-a321-452bd3f97c02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:29:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:30:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:25Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3aa67cc91dcc namespace: kserve-ci-e2e-test resourceVersion: "17147" uid: b12659e0-cf8a-42d6-bcaa-60f0df8d84b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:02Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3b273599dc0b namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 7da2eefa-258d-45a8-bfcc-e74b59cbe46f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8664e04d-5336-401b-9a09-6b159a33f1d7 kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:02Z" name: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7.18be3b27359a54da namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: c7f17f9c-4515-405d-b451-8f5b8aa00f5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-766f3-predictor-6b6855b58b namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 1ab6351e-ba0e-45c7-8b5e-c79a6fb93ab7 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'Created pod: error-404-isvc-766f3-predictor-6b6855b58b-dmhf7' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:47Z" name: error-404-isvc-766f3-predictor-6b6855b58b.18be3aa60861b2a8 namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: f3942688-fae5-4e80-b705-d3cc0314046f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 10e6b1ea-14d2-4346-8336-03dbbc0cd335 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Scaled up replica set error-404-isvc-766f3-predictor-6b6855b58b from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:47Z" name: error-404-isvc-766f3-predictor.18be3aa60514b97b namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: 062d8b74-71ac-4603-9d1b-e9a733745d35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 4b29f4d3-f858-4718-a316-1c719e99cdcb kind: Event lastTimestamp: "2026-07-01T17:30:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:17Z" name: error-404-isvc-766f3-predictor.18be3aa9839a00b9 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 1aa6ccb9-f7bb-4718-b52c-8544042316ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 4b29f4d3-f858-4718-a316-1c719e99cdcb kind: Event lastTimestamp: "2026-07-01T17:30:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:17Z" name: error-404-isvc-766f3-predictor.18be3aa983fd3d9c namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: f73b0ce1-a47c-45ea-ac0d-21a49ff2d233 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: 4b29f4d3-f858-4718-a316-1c719e99cdcb kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:02Z" name: error-404-isvc-766f3-predictor.18be3ab0815ea16f namespace: kserve-ci-e2e-test resourceVersion: "17477" uid: 11d9697a-22f6-4634-aea1-b3afbdb9852c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: 4b29f4d3-f858-4718-a316-1c719e99cdcb kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:02Z" name: error-404-isvc-766f3-predictor.18be3ab081c39e89 namespace: kserve-ci-e2e-test resourceVersion: "17478" uid: 3e36395f-a29e-4aa4-b47b-458cf4f0a4d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-766f3 namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: f8c0bdf3-f3c4-4d76-b950-efb79d81dc68 kind: Event lastTimestamp: "2026-07-01T17:30:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:35Z" name: error-404-isvc-766f3.18be3aa605281e7a namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: fecd9d76-b21d-4c1f-bc26-b531bb148ad1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-766f3 namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: f8c0bdf3-f3c4-4d76-b950-efb79d81dc68 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-766f3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-766f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:47Z" name: error-404-isvc-766f3.18be3aa605bad00d namespace: kserve-ci-e2e-test resourceVersion: "16738" uid: 39e7b011-52fc-442b-bf48-495529a1f419 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-766f3 namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: f8c0bdf3-f3c4-4d76-b950-efb79d81dc68 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-766f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:47Z" name: error-404-isvc-766f3.18be3aa605bb26d4 namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: 1567c46a-2e62-4d62-a2e5-0782677541bf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-766f3 namespace: kserve-ci-e2e-test resourceVersion: "17244" uid: f8c0bdf3-f3c4-4d76-b950-efb79d81dc68 kind: Event lastTimestamp: "2026-07-01T17:30:35Z" message: InferenceService [error-404-isvc-766f3] is Ready metadata: creationTimestamp: "2026-07-01T17:30:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:35Z" name: error-404-isvc-766f3.18be3ab13bf5e37b namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: 6720ad0f-52f7-4b35-89dc-43bc811d8d31 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:40:12.293157Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "21999" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:40:12Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b378ef863ef namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 70236632-c6e4-4db4-b9a7-22afd83cdd13 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8b57f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:12Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b379aad99c8 namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 57ccaf47-fdd7-4314-8427-13c211b1223a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:40:13Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b37cc49f19b namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 874908b7-36d0-4230-aa53-48b733896bb7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b37cdd5bc7f namespace: kserve-ci-e2e-test resourceVersion: "22051" uid: d2bbb795-4613-4f73-bf00-db66830e0864 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b37d0b2f8ba namespace: kserve-ci-e2e-test resourceVersion: "22053" uid: ddf2b98c-55f8-4643-acf6-2d855695ada8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b37d1443472 namespace: kserve-ci-e2e-test resourceVersion: "22055" uid: 2e376073-8776-4a89-a865-b36c855f48de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b37d14f2e28 namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: d419eccb-9d91-46c1-b421-28a2f0aa1821 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b37d40f47aa namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: 09fde89f-8569-4d6d-a7b7-c087cd8c1044 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b37d4e77478 namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: 957cd89e-8533-42ef-9a2e-fa4c5cb95389 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:40:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:40:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:50Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3b3821f2bcd1 namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: 7a969d40-811f-43f6-a4d9-5aa81338bf8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3bbb12b3ee88 namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: 5c8f4197-4cfd-47ad-8148-92844e236a0d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: e25145e5-82ef-4c78-8e97-84a50628dc1a kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd.18be3bbb12b45120 namespace: kserve-ci-e2e-test resourceVersion: "26669" uid: 8ca39b95-4713-432e-9350-6db14079871b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8b57f-predictor-5c84bc7466 namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: c6d395dd-004f-4a8d-9037-6a9ef928cf69 kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: 'Created pod: error-404-isvc-8b57f-predictor-5c84bc7466-fn2rd' metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:12Z" name: error-404-isvc-8b57f-predictor-5c84bc7466.18be3b378e8ad4b7 namespace: kserve-ci-e2e-test resourceVersion: "22001" uid: befd84ea-8959-4449-b175-a51c6409fd8e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21989" uid: d4a52d66-5003-4694-bd0f-77d84818aaec kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: Scaled up replica set error-404-isvc-8b57f-predictor-5c84bc7466 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:12Z" name: error-404-isvc-8b57f-predictor.18be3b378cf0a4d0 namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: c5f4ab02-fc67-4523-9541-a15b373ae7f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21990" uid: 583aa8e5-886a-4778-bab4-d2e8cea7955c kind: Event lastTimestamp: "2026-07-01T17:40:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:27Z" name: error-404-isvc-8b57f-predictor.18be3b3b0b7781e7 namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: 3099a3b9-3345-425d-84c5-ac7b5b3a3e3e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21990" uid: 583aa8e5-886a-4778-bab4-d2e8cea7955c kind: Event lastTimestamp: "2026-07-01T17:40:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:27Z" name: error-404-isvc-8b57f-predictor.18be3b3b0bd9631f namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 484aa29e-f574-4829-a158-786223c736a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "22230" uid: 583aa8e5-886a-4778-bab4-d2e8cea7955c kind: Event lastTimestamp: "2026-07-01T17:41:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:42Z" name: error-404-isvc-8b57f-predictor.18be3b3e8a82a988 namespace: kserve-ci-e2e-test resourceVersion: "22860" uid: 74b4af15-cab1-4888-8736-9a677a2aff35 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "22230" uid: 583aa8e5-886a-4778-bab4-d2e8cea7955c kind: Event lastTimestamp: "2026-07-01T17:41:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:42Z" name: error-404-isvc-8b57f-predictor.18be3b3e8aee07c0 namespace: kserve-ci-e2e-test resourceVersion: "22861" uid: 04628ba5-b865-4205-ae8b-c151881aa692 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b57f namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: c21085f5-a094-429b-b36f-c2cb45c6caea kind: Event lastTimestamp: "2026-07-01T17:41:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:41:00Z" name: error-404-isvc-8b57f.18be3b378cfc6343 namespace: kserve-ci-e2e-test resourceVersion: "22521" uid: a6ee540c-85b6-4d72-bf76-18255f771da1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b57f namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: c21085f5-a094-429b-b36f-c2cb45c6caea kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-8b57f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8b57f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:12Z" name: error-404-isvc-8b57f.18be3b378d7c771d namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 2756c82d-d475-459a-a8f0-c1eb4eb39b70 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b57f namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: c21085f5-a094-429b-b36f-c2cb45c6caea kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8b57f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:12Z" name: error-404-isvc-8b57f.18be3b378d7cdfeb namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 4d84f960-0210-48f4-b8d5-28242181743f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8b57f namespace: kserve-ci-e2e-test resourceVersion: "22519" uid: c21085f5-a094-429b-b36f-c2cb45c6caea kind: Event lastTimestamp: "2026-07-01T17:41:00Z" message: InferenceService [error-404-isvc-8b57f] is Ready metadata: creationTimestamp: "2026-07-01T17:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:41:00Z" name: error-404-isvc-8b57f.18be3b42dd59021e namespace: kserve-ci-e2e-test resourceVersion: "22520" uid: 74199dea-c4ed-4da9-ae34-0e4b1059ce94 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:39:02.074848Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:39:02Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b2735a1d221 namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: a87c302f-91db-49ac-9ff9-848f88aebc51 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-98f1c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:02Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b274113eb47 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 64169542-460b-47eb-8fff-01498fb3ad56 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:39:03Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:39:03Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b27735386b7 namespace: kserve-ci-e2e-test resourceVersion: "21074" uid: 65d9523e-6521-44cf-905e-79c25fb0db9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:39:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:03Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b2774c6c1a0 namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: f676b58f-ed8c-47cf-874b-0bad5490cd22 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:39:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:03Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b2777cec3f4 namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: b686a419-c013-4d2d-87a9-fb924b31225a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:39:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:03Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b27783db96c namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 2f3ad200-d87b-494b-bc1b-c481d43c8b14 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:39:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:03Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b2778481969 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: b49bab11-c8ae-4942-b30c-90197eb15b8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:39:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:03Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b277b3a5a00 namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: b92b51dc-7fe5-4947-a2bf-519bfad50e59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:39:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:03Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b277ba8b3e2 namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: f374ba0c-102d-4a40-beb7-b358a980f67b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:39:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:40Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b27c3e0236b namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: e5811981-648f-4280-8d1a-2d3b762d5f16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:12Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b379166d093 namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: cf4df5c2-6598-41f9-a3c3-895deb0127f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:12Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b3791676f63 namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: e91c7e88-fcd5-45fc-9da9-93ebeb21dbe8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 92a915eb-bc3c-4e50-8b6b-48f7de29d84f kind: Event lastTimestamp: "2026-07-01T17:40:15Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:15Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp.18be3b384c2434b6 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: 88200afa-5a54-49f9-9633-b783446b55cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-98f1c-predictor-666bcdd5dd namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: ce1a7ed6-56c4-46a2-ad5f-847481f79497 kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: 'Created pod: error-404-isvc-98f1c-predictor-666bcdd5dd-nxjqp' metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:02Z" name: error-404-isvc-98f1c-predictor-666bcdd5dd.18be3b273521b299 namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: 82bf428e-4eb2-4d1d-803e-579f31ebfbd7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-98f1c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 4d86f53d-e98c-419d-ae7e-5a7b921786e5 kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: Scaled up replica set error-404-isvc-98f1c-predictor-666bcdd5dd from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:02Z" name: error-404-isvc-98f1c-predictor.18be3b27332af90c namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: ee7be3ad-8943-4338-8dc7-fcd351d634e6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:39:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-98f1c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: b20c6684-f5bc-44ab-b61f-1c740bda0e61 kind: Event lastTimestamp: "2026-07-01T17:40:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:02Z" name: error-404-isvc-98f1c-predictor.18be3b2ab28e977f namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 62c1cfbe-e3b5-46c4-99f6-67c19b3f4206 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:39:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-98f1c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: b20c6684-f5bc-44ab-b61f-1c740bda0e61 kind: Event lastTimestamp: "2026-07-01T17:40:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:02Z" name: error-404-isvc-98f1c-predictor.18be3b2ab3058069 namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: 46ecb970-c53f-4b0c-bd86-75289f22d0a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-98f1c namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: c77a5bba-aa1b-448f-952d-2ef46ad8ef54 kind: Event lastTimestamp: "2026-07-01T17:39:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:50Z" name: error-404-isvc-98f1c.18be3b27332ad265 namespace: kserve-ci-e2e-test resourceVersion: "21708" uid: 43075eb2-16de-491f-8b61-771ff9c16fd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-98f1c namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: c77a5bba-aa1b-448f-952d-2ef46ad8ef54 kind: Event lastTimestamp: "2026-07-01T17:39:50Z" message: InferenceService [error-404-isvc-98f1c] is Ready metadata: creationTimestamp: "2026-07-01T17:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:50Z" name: error-404-isvc-98f1c.18be3b328152aaeb namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: b0564c3e-441f-4a93-a41c-f0796d9d43b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:39:26.593004Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c4578-predictor-8585647f56-4hsc9 to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:39:26Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3b2ceb073f90 namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 296fbe21-fbe5-4abc-8265-d525818b9949 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:39:27Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:39:27Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3b2d047ad218 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: cde4558a-18f7-43e7-b6ca-4197ed7ed579 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:39:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:27Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3b2d0610751b namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: 42abe4b5-ab5f-4770-b13e-23e7ba3f4192 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:39:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:27Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3b2d094a039b namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: a5fee545-fd0c-4cc1-9234-b726da712fa0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:39:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:27Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3b2d09c4c79d namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 577eae76-4b5c-4dd3-bd5c-e2ae2af124bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:39:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:27Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3b2d09cd6026 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: ec65c21e-2539-4ff9-9216-e0804d44981f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:39:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:27Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3b2d0ca81dfa namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: a82d9805-a96f-41e4-b47b-1c0de0fb97b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:39:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:27Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3b2d0d1c396c namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: e21ce0b2-4da4-42a6-845b-7dd40f908cee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:39:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:48:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:44Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3b2d60789481 namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: e4de70db-80f5-4748-af07-1733c455d47a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3bae222c28dd namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: d8eb1cac-5ece-43e0-850e-457354deab3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3bae222caf72 namespace: kserve-ci-e2e-test resourceVersion: "26111" uid: c52b0f31-f92d-4f19-8fb7-8580d4069bce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c4578-predictor-8585647f56-4hsc9 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 87e604e1-c30c-47f6-bd5a-aab6e79f83da kind: Event lastTimestamp: "2026-07-01T17:48:44Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:44Z" name: error-404-isvc-c4578-predictor-8585647f56-4hsc9.18be3baed509bc10 namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 0e1ec851-a0ce-4d8d-9ef7-e53c2dcbec4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c4578-predictor-8585647f56 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: dfc2d6b7-4d05-4bd3-8e8a-841d9e1850c5 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: 'Created pod: error-404-isvc-c4578-predictor-8585647f56-4hsc9' metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:26Z" name: error-404-isvc-c4578-predictor-8585647f56.18be3b2cea946315 namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 5074af34-ea55-4923-b84a-13ac68884029 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: 291591f7-b393-41c9-9102-78d1195f73a1 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Scaled up replica set error-404-isvc-c4578-predictor-8585647f56 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:26Z" name: error-404-isvc-c4578-predictor.18be3b2ce8766f47 namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: 859ca81c-9d52-4f23-a375-56983667d5ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:39:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: c7b8ce25-c86e-41e1-86f9-037c07b2a5a5 kind: Event lastTimestamp: "2026-07-01T17:39:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:56Z" name: error-404-isvc-c4578-predictor.18be3b3066c06541 namespace: kserve-ci-e2e-test resourceVersion: "21751" uid: 3865198e-9947-4315-9413-fa0387597ad8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:39:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: c7b8ce25-c86e-41e1-86f9-037c07b2a5a5 kind: Event lastTimestamp: "2026-07-01T17:39:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:56Z" name: error-404-isvc-c4578-predictor.18be3b3067123ebb namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: 5b751c24-6848-49a6-8138-66c511a5d04f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: c7b8ce25-c86e-41e1-86f9-037c07b2a5a5 kind: Event lastTimestamp: "2026-07-01T17:40:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:41Z" name: error-404-isvc-c4578-predictor.18be3b37646e7d6e namespace: kserve-ci-e2e-test resourceVersion: "22357" uid: d3b606b8-ed6a-4f82-8cd0-b49eaa66af5a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: c7b8ce25-c86e-41e1-86f9-037c07b2a5a5 kind: Event lastTimestamp: "2026-07-01T17:40:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:41Z" name: error-404-isvc-c4578-predictor.18be3b3764d2845a namespace: kserve-ci-e2e-test resourceVersion: "22358" uid: e3e692ae-4611-46c2-9dd7-87afb7408831 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c4578 namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: 9dbfc978-3c11-4c24-8fd0-8fe5a5f21d75 kind: Event lastTimestamp: "2026-07-01T17:40:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:14Z" name: error-404-isvc-c4578.18be3b2ce87daac8 namespace: kserve-ci-e2e-test resourceVersion: "22093" uid: 5e5e3cac-48b4-4052-be73-46f7625b5c66 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c4578 namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 9dbfc978-3c11-4c24-8fd0-8fe5a5f21d75 kind: Event lastTimestamp: "2026-07-01T17:40:14Z" message: InferenceService [error-404-isvc-c4578] is Ready metadata: creationTimestamp: "2026-07-01T17:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:14Z" name: error-404-isvc-c4578.18be3b381c4ad1a0 namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: dd81fa40-671d-4a6b-b61a-630364dee7a1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:27:27.267440Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d0081-predictor-754fc45f7b-pkk58 to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:27:27Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a856fe223e1 namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: aa252df7-38e1-42f4-b474-102f1ae908ba reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d0081-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:27Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a857b7f0353 namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 592f91a5-4b1a-4e46-bd8a-2ede8b93a841 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:29Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:27:29Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a85e2e942c6 namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: 4bcb2510-af14-4be4-96b5-d911054c4462 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:29Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" metadata: creationTimestamp: "2026-07-01T17:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:29Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a85e5d9f462 namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: bbe02b2b-53c5-47a0-9ea0-5a79814ebf65 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:40Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1692" in 11.689s (11.689s including waiting). Image size: 1334626922 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:40Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a889e940f62 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: fbb6d35b-9c7c-4547-a61a-b9071453a057 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:40Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a88a1f14579 namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: cf0baaa0-a981-451f-8828-21c3ab1ed219 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:41Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a88a26538c9 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 302b0970-f8d6-4e8f-bb88-7c5620f7254d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:41Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a88a2724076 namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: 8b4fcd46-0fe5-475a-a604-ab2cceeb3f41 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:43Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.278s (2.278s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:43Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a892a46dcba namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: 0e7b9f82-49a3-4bcc-b422-57c8e2924d0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:43Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a892d14221b namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: 21711133-c2d6-4424-a5f3-a868609b54f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:27:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:43Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a892dc196a2 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: cc1b5c97-26f6-411c-93a1-de8cbddbb7a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:27:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:28:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:28:20Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a89827d1459 namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: ffb8e46e-dba9-4960-945a-d1cbf3e4bcd7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a9b55f0a125 namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 78035b62-d8cd-46c5-8439-d6de8d974c86 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 21d5fd76-11a2-4651-8eb6-ec9c022e59d3 kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: error-404-isvc-d0081-predictor-754fc45f7b-pkk58.18be3a9b55f0fe9d namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 26104c2a-7ab8-4ef2-9ab9-848f656b67bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d0081-predictor-754fc45f7b namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: d1b4d2ec-b6d2-4af9-ad71-8f8e82692be2 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: 'Created pod: error-404-isvc-d0081-predictor-754fc45f7b-pkk58' metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:27Z" name: error-404-isvc-d0081-predictor-754fc45f7b.18be3a856f7d0bba namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: 36f91e10-3f8d-466e-b0a6-bbddcfbf8ef2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: f042a047-f5f3-4529-a16f-d795c3076b81 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Scaled up replica set error-404-isvc-d0081-predictor-754fc45f7b from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:27Z" name: error-404-isvc-d0081-predictor.18be3a856d60d743 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: d8a1b4ff-d5b4-4ac1-ba09-f50be838903a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: dfe857a9-d6ac-401a-af49-f54a0443bf67 kind: Event lastTimestamp: "2026-07-01T17:28:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:27Z" name: error-404-isvc-d0081-predictor.18be3a88ebf88b33 namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 5874f596-cebb-45de-807d-92de7bb5381c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: dfe857a9-d6ac-401a-af49-f54a0443bf67 kind: Event lastTimestamp: "2026-07-01T17:28:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:27Z" name: error-404-isvc-d0081-predictor.18be3a88ec5e9e14 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 62cf4f05-41e8-47c5-8f16-0f565090abfb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:28:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: dfe857a9-d6ac-401a-af49-f54a0443bf67 kind: Event lastTimestamp: "2026-07-01T17:28:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:57Z" name: error-404-isvc-d0081-predictor.18be3a96e71357fc namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 78e30b25-99f7-4704-aec3-5e9824ea6d51 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:28:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: dfe857a9-d6ac-401a-af49-f54a0443bf67 kind: Event lastTimestamp: "2026-07-01T17:28:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:57Z" name: error-404-isvc-d0081-predictor.18be3a96e7790557 namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 5284a6ec-58cf-4c8e-939c-78e7c75299ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0081 namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 8ee8a055-ffa9-409f-9f1e-dc094277c157 kind: Event lastTimestamp: "2026-07-01T17:28:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:28:30Z" name: error-404-isvc-d0081.18be3a856d6b81e6 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: a1297ec1-1f67-4b1d-b261-b01a49758c16 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0081 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 8ee8a055-ffa9-409f-9f1e-dc094277c157 kind: Event lastTimestamp: "2026-07-01T17:28:30Z" message: InferenceService [error-404-isvc-d0081] is Ready metadata: creationTimestamp: "2026-07-01T17:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:28:30Z" name: error-404-isvc-d0081.18be3a943da780e6 namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: 6aa859e5-d153-4711-a348-b8aa52af7415 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:27:26.901444Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 to ip-10-0-129-36.ec2.internal metadata: creationTimestamp: "2026-07-01T17:27:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:27:26Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a855a117f35 namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: f122988f-53aa-4a2b-8553-a73894f8c093 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:27:27Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8573a1a34c namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: bc4b2054-8abb-4749-9113-2f6c21542aea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:18a74e28e595ffa9dfe5870010eb9e94a07ca208b9ee33486e4d6bf5e7fb5215" already present on machine metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:27Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a857504df98 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 6cff3416-1253-4b92-9f29-8e2d3a1122c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:27Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8577fe5f96 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: bd7ecb77-e3d2-48f8-93be-0ddf6f92dfab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:27Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a857872db22 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: 5f3d7d48-ea1e-47fc-bf1f-a907b8ed01ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:31Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1692" metadata: creationTimestamp: "2026-07-01T17:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:31Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a866b7cf17f namespace: kserve-ci-e2e-test resourceVersion: "15184" uid: 09648d72-7910-4f44-ad0f-cf4ba3285574 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:54Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1692" in 22.923s (22.923s including waiting). Image size: 1555968947 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:54Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8bc1d8fac7 namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: ab445e1a-0ea7-44a1-9fbf-b56f8410ddf7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:54Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8bc7dc976f namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: 44ce8a78-8319-4117-82f7-fa0a1b4e09f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:54Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8bc918ae35 namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: 35356c81-4a56-44fb-9b02-361025fad5f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:54Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8bc926ea1f namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: a0ccd2f1-9eee-41e8-bc37-97048fb3e839 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:57Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.533s (2.533s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:57Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8c602b130f namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 33a24856-e913-46e2-bcdc-511e551250e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:57Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8c6472b36e namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: e2a6534b-69fd-4c4e-881f-a24e05a529b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:27:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:57Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8c64d936c1 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: 7e2cdcc8-7bba-4315-8f3f-220007bba844 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T17:27:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:29:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:04Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3a8cbee0d3bc namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 8b7dc0cd-3fed-4d46-8cf6-28917bf7b4c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3aa5ffa7fbed namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: fd38c766-0124-4c67-96c0-9057fadaf038 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3aa5ffa86357 namespace: kserve-ci-e2e-test resourceVersion: "16699" uid: 2cb36add-bd56-4dfd-bb30-3bdc4a6d01a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 654e3775-925a-4880-9245-877bc17515d5 kind: Event lastTimestamp: "2026-07-01T17:29:49Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:49Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8.18be3aa6973e4874 namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: 5cd8c130-fbfd-41b1-9def-0ae9856f2b68 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6474cd55c7 namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: fe3f225b-be22-44ac-91a9-ae7633035c19 kind: Event lastTimestamp: "2026-07-01T17:27:26Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6474cd55c7-g4tp8' metadata: creationTimestamp: "2026-07-01T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:26Z" name: isvc-sklearn-graph-1-predictor-6474cd55c7.18be3a85598968dc namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: 22add901-e35a-446f-9aed-a0f6c1a5e7e7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: 7e706183-1ceb-4efa-b92a-6c33e903ba92 kind: Event lastTimestamp: "2026-07-01T17:27:26Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6474cd55c7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:26Z" name: isvc-sklearn-graph-1-predictor.18be3a8554e3aaef namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: a88531ae-2056-4dc9-b8e3-06631283ce5a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 392be3db-690e-4cc6-b636-d82610b90342 kind: Event lastTimestamp: "2026-07-01T17:28:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:11Z" name: isvc-sklearn-graph-1-predictor.18be3a88da66d3d1 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: ad77bc31-793b-4c0f-a380-75b4cc795e5b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 392be3db-690e-4cc6-b636-d82610b90342 kind: Event lastTimestamp: "2026-07-01T17:28:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:11Z" name: isvc-sklearn-graph-1-predictor.18be3a88dacda20d namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: 97bf14a8-e21a-4738-84da-c6dcef6e7483 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 392be3db-690e-4cc6-b636-d82610b90342 kind: Event lastTimestamp: "2026-07-01T17:29:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:42Z" name: isvc-sklearn-graph-1-predictor.18be3a935769bb01 namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: 081e2d9f-a366-4f31-b674-d2e1f1510354 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 392be3db-690e-4cc6-b636-d82610b90342 kind: Event lastTimestamp: "2026-07-01T17:29:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:42Z" name: isvc-sklearn-graph-1-predictor.18be3a9357ae12d1 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 07f25c00-0b99-4e62-a4b5-18e7fc36d467 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T17:27:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: c57fd423-5804-4445-a4cc-92a88e4112e9 kind: Event lastTimestamp: "2026-07-01T17:29:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:14Z" name: isvc-sklearn-graph-1.18be3a855b752a26 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 0d7af363-74d4-47cc-99bc-ac1b1f14b8cc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: c57fd423-5804-4445-a4cc-92a88e4112e9 kind: Event lastTimestamp: "2026-07-01T17:27:26Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:27:26Z" name: isvc-sklearn-graph-1.18be3a855be7309c namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: d9b1978d-8ed1-490a-8283-dd4aa1b62d1f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: c57fd423-5804-4445-a4cc-92a88e4112e9 kind: Event lastTimestamp: "2026-07-01T17:27:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:27:26Z" name: isvc-sklearn-graph-1.18be3a855be77bd8 namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 1515ccdc-d974-4f20-9eb0-c4da53b67ae3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: c57fd423-5804-4445-a4cc-92a88e4112e9 kind: Event lastTimestamp: "2026-07-01T17:29:14Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-01T17:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:14Z" name: isvc-sklearn-graph-1.18be3a9e7754d67a namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: b31a67b4-143d-44d8-a920-68cceb5b96a4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:27:27.386901Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:27:27Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a857700fe33 namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: 79592265-3518-4c4b-9ffc-ec6759cd6b73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:27:27Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a859269f218 namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: 703fe008-3be0-4444-8700-c6ae783bbee6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:18a74e28e595ffa9dfe5870010eb9e94a07ca208b9ee33486e4d6bf5e7fb5215" metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:27Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a8593c0fb38 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 14f4cadf-f843-481d-b3b3-733a2f619c54 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:33Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:18a74e28e595ffa9dfe5870010eb9e94a07ca208b9ee33486e4d6bf5e7fb5215" in 6.103s (6.103s including waiting). Image size: 301878739 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:33Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a86ff86d886 namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 707ea608-e175-445b-ae47-8d9ecb515582 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:34Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a87027ff40c namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: 69c387cd-1dd2-4fe6-9906-f806955d41b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:27:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:34Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a8702eae140 namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: ec422ec2-40a1-475c-b23c-4aee773f1bdf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:41Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1692" metadata: creationTimestamp: "2026-07-01T17:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:41Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a88b555fdba namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: 8b8cac45-c821-47cd-9806-c83ffe005337 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:48Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1692" in 6.969s (6.969s including waiting). Image size: 1555968947 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:48Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a8a54c1fd63 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 02df2a69-c396-4ca8-99a7-2ca1d7c4f405 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:48Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a8a583987f5 namespace: kserve-ci-e2e-test resourceVersion: "15360" uid: 91801ce2-7f2b-4300-915b-ef4b6fde980c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:48Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a8a58ac0e5f namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: d930db83-ebb0-4539-8d1b-7b314009424a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:48Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a8a58bffa9b namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: c077b118-0543-4242-97c3-aac483b63abb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:48Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a8a5c3d1975 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: 31c2cabf-eb6f-432a-9dec-7aed346fdc24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:27:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:48Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a8a5ca96b12 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 66410904-9785-4c32-8deb-bc9eab4f5c0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T17:27:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:27:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3a8ad10d8ddd namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: 818e4c2f-241f-419e-8527-29e73c3591d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3aa5fc9e1389 namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: 820292da-8274-4b99-b595-eb219ed0d3e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 7975c682-1ce4-4a39-8232-2db46ba88a63 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd.18be3aa5fc9e9cea namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 92c65cd0-f045-445d-91ce-6e17f87e8ee8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-9c6bdd8f5 namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: aed90590-cf68-4a8d-9862-638957bc07c3 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-9c6bdd8f5-2r4nd' metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:27Z" name: isvc-sklearn-graph-2-predictor-9c6bdd8f5.18be3a857695289d namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: df5178f5-9062-4d32-a1e6-06b375ef313c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 3e5fa4fb-5a36-4d35-86f0-fdc69d51ea0a kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-9c6bdd8f5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:27Z" name: isvc-sklearn-graph-2-predictor.18be3a8573d8264b namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: aeeafbdd-fc11-4ec9-acb9-3257dcb1c128 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15082" uid: e5edf637-127c-4db4-8c60-e04574e87efd kind: Event lastTimestamp: "2026-07-01T17:28:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:27Z" name: isvc-sklearn-graph-2-predictor.18be3a88f2631bea namespace: kserve-ci-e2e-test resourceVersion: "15716" uid: 5f588a11-b03b-4b2c-bd58-0ef71c41a3a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15082" uid: e5edf637-127c-4db4-8c60-e04574e87efd kind: Event lastTimestamp: "2026-07-01T17:28:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:27Z" name: isvc-sklearn-graph-2-predictor.18be3a88f2ce2a74 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 99b31852-1318-42c1-8a4d-4b3e1cdc5f54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:28:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: e5edf637-127c-4db4-8c60-e04574e87efd kind: Event lastTimestamp: "2026-07-01T17:29:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:42Z" name: isvc-sklearn-graph-2-predictor.18be3a96edcbf5bd namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 50cc0396-bf96-4394-9614-1d4a2721dbc5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:28:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: e5edf637-127c-4db4-8c60-e04574e87efd kind: Event lastTimestamp: "2026-07-01T17:29:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:42Z" name: isvc-sklearn-graph-2-predictor.18be3a96ee378e19 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: 351f60be-2210-436b-8c66-0cdd496d26c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8505aae4-bd16-4021-af28-e72759e0e867 kind: Event lastTimestamp: "2026-07-01T17:29:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:07Z" name: isvc-sklearn-graph-2.18be3a8573ed3064 namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: 9db6c309-f471-49d6-9762-8358840fe9a1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 8505aae4-bd16-4021-af28-e72759e0e867 kind: Event lastTimestamp: "2026-07-01T17:29:07Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-01T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:07Z" name: isvc-sklearn-graph-2.18be3a9cce112a47 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 9c807244-796d-4c9b-bf7b-67d1417785cd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:27:27.182755Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-hxghh to ip-10-0-129-36.ec2.internal metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:27:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a856ad5ecdd namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 5731417a-d50c-4904-8b5b-7001a125de7c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:27:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8584c5b0bd namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 577635b0-3ab3-4bb0-b4e5-b22995ced543 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:18a74e28e595ffa9dfe5870010eb9e94a07ca208b9ee33486e4d6bf5e7fb5215" already present on machine metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8586151b5a namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: c8f290da-f21a-4fba-bacb-9873a6c32b8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8588fb1b7a namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 6519487f-69be-46c0-8936-2db80bfe7a58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a85896b14c9 namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: 762bd7aa-f051-43b1-a6a6-2d0d2f017d53 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:31Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T17:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a866b95eb07 namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 917c73c7-421d-4a2e-9675-81ef7a35b2cd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:54Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.345s (23.345s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8bdb149248 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: e43223f1-52d6-40d4-b216-d8a390a20fc6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8be0fadb14 namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: b684a818-ecbd-409a-a447-2c6941f81f90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8be1ce0bf2 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: d4ed7cf6-547a-4443-9469-ecda462c98c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8be1da9830 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: eee0f6de-e281-4d05-884b-abfdb3274b89 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:57Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.13s (2.13s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8c60d26341 namespace: kserve-ci-e2e-test resourceVersion: "15445" uid: f815ef94-1d04-4f66-9456-3d23f361e0b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8c64a493c7 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 09360e5f-83d4-47c0-9ae1-87238382dded reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:27:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8c650c39b7 namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: f39cb41d-0966-4826-9f23-6ebfa62f6d64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:27:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:28:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:28:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3a8cbee1ce27 namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: c5412c23-014d-4a9c-b8c8-da2e4a50364e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3aa6050c4575 namespace: kserve-ci-e2e-test resourceVersion: "16733" uid: d9a63564-c710-40c0-95f9-b293639278d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3aa6050ff917 namespace: kserve-ci-e2e-test resourceVersion: "16737" uid: cb19e7f4-1ea4-423a-af9e-3c0850ec7f53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hxghh namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: bee89f35-320b-464f-a008-6e266da12577 kind: Event lastTimestamp: "2026-07-01T17:29:49Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-hxghh.18be3aa6973fc71c namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: 3e3b5d17-9c44-45cd-82ff-db966952dc47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-36.ec2.internal source: component: kubelet host: ip-10-0-129-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 4be9c42d-246a-4faf-bc00-d8781407f8e1 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-hxghh' metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:27Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be3a856a666b7c namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 44daf4ec-d5de-455d-bf75-d47e2198f86b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: d8d8da92-6883-495f-9a4c-e8f575bb202e kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:27Z" name: isvc-xgboost-graph-predictor.18be3a856777f5a0 namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: 2f8548a0-f94b-40b2-9bb7-818e529f2bb6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: 75482585-b2e4-44cf-baf8-e0a1021b60bf kind: Event lastTimestamp: "2026-07-01T17:28:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:42Z" name: isvc-xgboost-graph-predictor.18be3a88e5fb3c48 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: 0aa17afb-83f6-4046-851e-71e8f01a516f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: 75482585-b2e4-44cf-baf8-e0a1021b60bf kind: Event lastTimestamp: "2026-07-01T17:28:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:42Z" name: isvc-xgboost-graph-predictor.18be3a88e665eaba namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: ade1b124-e6c4-43fb-bddf-e8fee7ea82dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:28:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 75482585-b2e4-44cf-baf8-e0a1021b60bf kind: Event lastTimestamp: "2026-07-01T17:29:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:42Z" name: isvc-xgboost-graph-predictor.18be3a93626689d5 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 38b8e695-8ae7-41ae-805b-5c330237b52e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:28:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 75482585-b2e4-44cf-baf8-e0a1021b60bf kind: Event lastTimestamp: "2026-07-01T17:29:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:42Z" name: isvc-xgboost-graph-predictor.18be3a9362ca286b namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: ce6e32e4-b85c-41f1-8e7c-eec6627a4939 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: a5e6f03c-a5f0-4d58-a11f-6bbc6260b914 kind: Event lastTimestamp: "2026-07-01T17:28:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:28:54Z" name: isvc-xgboost-graph.18be3a856782d575 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: df2d9d23-9501-47f8-9d56-45a1471059a9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: a5e6f03c-a5f0-4d58-a11f-6bbc6260b914 kind: Event lastTimestamp: "2026-07-01T17:28:54Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-01T17:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:28:54Z" name: isvc-xgboost-graph.18be3a99cea08d98 namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: 84e01528-9956-443f-a37e-e620d016fafa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:29:36.847379Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5cf9f9c9dc-n4nq6 namespace: kserve-ci-e2e-test resourceVersion: "16503" uid: 241b85d7-2d84-4b8a-9362-09ffbaa76939 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5cf9f9c9dc-n4nq6 to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:29:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:29:36Z" name: model-chainer-5cf9f9c9dc-n4nq6.18be3aa39b73198b namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 4ab2c8a8-183d-44bc-9836-b460acd46898 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:36Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5cf9f9c9dc-n4nq6 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 241b85d7-2d84-4b8a-9362-09ffbaa76939 kind: Event lastTimestamp: "2026-07-01T17:29:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:36Z" name: model-chainer-5cf9f9c9dc-n4nq6.18be3aa3a44daedb namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: 6ec0903d-cf81-4e4b-8f8b-0636225a458b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:37Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5cf9f9c9dc-n4nq6 namespace: kserve-ci-e2e-test resourceVersion: "16513" uid: 241b85d7-2d84-4b8a-9362-09ffbaa76939 kind: Event lastTimestamp: "2026-07-01T17:29:37Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:29:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:29:37Z" name: model-chainer-5cf9f9c9dc-n4nq6.18be3aa3d877ecf7 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 3e93dcb2-8fd7-4f36-9f78-16e084d484b6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cf9f9c9dc-n4nq6 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 241b85d7-2d84-4b8a-9362-09ffbaa76939 kind: Event lastTimestamp: "2026-07-01T17:29:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:37Z" name: model-chainer-5cf9f9c9dc-n4nq6.18be3aa3d9c82419 namespace: kserve-ci-e2e-test resourceVersion: "16534" uid: eb8ea23e-30b5-470f-8e53-d7daf40f1f6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cf9f9c9dc-n4nq6 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 241b85d7-2d84-4b8a-9362-09ffbaa76939 kind: Event lastTimestamp: "2026-07-01T17:29:37Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-01T17:29:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:37Z" name: model-chainer-5cf9f9c9dc-n4nq6.18be3aa3dcdb5f7e namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: b79e8ef3-4db1-437e-8ecc-013ff2ed873c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cf9f9c9dc-n4nq6 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 241b85d7-2d84-4b8a-9362-09ffbaa76939 kind: Event lastTimestamp: "2026-07-01T17:29:37Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-01T17:29:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:37Z" name: model-chainer-5cf9f9c9dc-n4nq6.18be3aa3dd449114 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: 3143888d-8a25-4f79-96f9-1fe2a7b62368 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cf9f9c9dc-n4nq6 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 241b85d7-2d84-4b8a-9362-09ffbaa76939 kind: Event lastTimestamp: "2026-07-01T17:29:46Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-01T17:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:46Z" name: model-chainer-5cf9f9c9dc-n4nq6.18be3aa5f5b6ddf1 namespace: kserve-ci-e2e-test resourceVersion: "16643" uid: 13e5d6f2-392e-4edd-9786-d94a74e3256b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:29:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cf9f9c9dc-n4nq6 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 241b85d7-2d84-4b8a-9362-09ffbaa76939 kind: Event lastTimestamp: "2026-07-01T17:30:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:14Z" name: model-chainer-5cf9f9c9dc-n4nq6.18be3aa67ac84d38 namespace: kserve-ci-e2e-test resourceVersion: "17065" uid: 56cc8d01-09c5-45d2-8177-2552eddf3dc9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5cf9f9c9dc namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: 3c1e99da-1a0a-4fb6-ae6c-1a40894d343d kind: Event lastTimestamp: "2026-07-01T17:29:36Z" message: 'Created pod: model-chainer-5cf9f9c9dc-n4nq6' metadata: creationTimestamp: "2026-07-01T17:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:36Z" name: model-chainer-5cf9f9c9dc.18be3aa39ad744f1 namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: f81c1390-a248-415c-a825-28cffdfc8773 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: 7f39d65a-ed68-4613-b561-dace6a811bfa kind: Event lastTimestamp: "2026-07-01T17:29:36Z" message: Scaled up replica set model-chainer-5cf9f9c9dc from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:36Z" name: model-chainer.18be3aa39a028ac4 namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: a4797ebe-cbe9-40b7-866f-8bcb442a0a05 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: f693a87d-d039-45b9-a344-d2250cf500be kind: Event lastTimestamp: "2026-07-01T17:29:36Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-01T17:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:36Z" name: model-chainer.18be3aa39e729291 namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: e4026edd-87ac-4507-b354-1be583ea431e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: f693a87d-d039-45b9-a344-d2250cf500be kind: Event lastTimestamp: "2026-07-01T17:29:36Z" message: 'Failed to update status for InferenceGraph "model-chainer": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:36Z" name: model-chainer.18be3aa3a0e5b023 namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: b2744aea-2022-4f1b-abf4-ad3193023a86 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: f693a87d-d039-45b9-a344-d2250cf500be kind: Event lastTimestamp: "2026-07-01T17:29:36Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:36Z" name: model-chainer.18be3aa3a0e5f621 namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: 13754ef6-a6e2-473f-8a81-00dee40ccce0 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:30:47.214423Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-766f3-f9786597b-xhzhn namespace: kserve-ci-e2e-test resourceVersion: "17339" uid: fe90fe7b-f8fd-4fcb-8962-f9a22544d016 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-766f3-f9786597b-xhzhn to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:30:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:30:47Z" name: sequence-graph-766f3-f9786597b-xhzhn.18be3ab3fda6f9da namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 481b8f42-2775-40c0-82d9-08309cb21037 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:47Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-766f3-f9786597b-xhzhn namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: fe90fe7b-f8fd-4fcb-8962-f9a22544d016 kind: Event lastTimestamp: "2026-07-01T17:30:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-766f3-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:47Z" name: sequence-graph-766f3-f9786597b-xhzhn.18be3ab40a160831 namespace: kserve-ci-e2e-test resourceVersion: "17360" uid: 4ff8f75d-f730-48b2-bc1a-1884a636d9d9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:48Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-766f3-f9786597b-xhzhn namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: fe90fe7b-f8fd-4fcb-8962-f9a22544d016 kind: Event lastTimestamp: "2026-07-01T17:30:48Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:30:48Z" name: sequence-graph-766f3-f9786597b-xhzhn.18be3ab43aadf75c namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: c33cd9e5-c18a-4985-a9b3-de458c831ca7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-766f3} kind: Pod name: sequence-graph-766f3-f9786597b-xhzhn namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: fe90fe7b-f8fd-4fcb-8962-f9a22544d016 kind: Event lastTimestamp: "2026-07-01T17:30:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:48Z" name: sequence-graph-766f3-f9786597b-xhzhn.18be3ab43c08b63a namespace: kserve-ci-e2e-test resourceVersion: "17369" uid: e2c5c5dd-93f6-4408-82db-560a93bd4dd1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-766f3} kind: Pod name: sequence-graph-766f3-f9786597b-xhzhn namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: fe90fe7b-f8fd-4fcb-8962-f9a22544d016 kind: Event lastTimestamp: "2026-07-01T17:30:48Z" message: 'Created container: sequence-graph-766f3' metadata: creationTimestamp: "2026-07-01T17:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:48Z" name: sequence-graph-766f3-f9786597b-xhzhn.18be3ab43f10e32e namespace: kserve-ci-e2e-test resourceVersion: "17370" uid: 285fe6fc-81ea-4147-a0c7-db59c68a9fc9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-766f3} kind: Pod name: sequence-graph-766f3-f9786597b-xhzhn namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: fe90fe7b-f8fd-4fcb-8962-f9a22544d016 kind: Event lastTimestamp: "2026-07-01T17:30:48Z" message: Started container sequence-graph-766f3 metadata: creationTimestamp: "2026-07-01T17:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:48Z" name: sequence-graph-766f3-f9786597b-xhzhn.18be3ab43f795e0c namespace: kserve-ci-e2e-test resourceVersion: "17371" uid: 03051b32-960f-4632-a7d2-00cf8b12421a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-766f3} kind: Pod name: sequence-graph-766f3-f9786597b-xhzhn namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: fe90fe7b-f8fd-4fcb-8962-f9a22544d016 kind: Event lastTimestamp: "2026-07-01T17:39:01Z" message: Stopping container sequence-graph-766f3 metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:01Z" name: sequence-graph-766f3-f9786597b-xhzhn.18be3b2725bdd85d namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: 012fb1b8-3d14-4930-b5e8-795531719a8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-766f3} kind: Pod name: sequence-graph-766f3-f9786597b-xhzhn namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: fe90fe7b-f8fd-4fcb-8962-f9a22544d016 kind: Event lastTimestamp: "2026-07-01T17:39:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:29Z" name: sequence-graph-766f3-f9786597b-xhzhn.18be3b27c4db665d namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 569daa9a-016c-4741-afc1-f423520e39eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-766f3-f9786597b namespace: kserve-ci-e2e-test resourceVersion: "17335" uid: 4064e7e5-dd80-4505-87fd-1312f81851b0 kind: Event lastTimestamp: "2026-07-01T17:30:47Z" message: 'Created pod: sequence-graph-766f3-f9786597b-xhzhn' metadata: creationTimestamp: "2026-07-01T17:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:47Z" name: sequence-graph-766f3-f9786597b.18be3ab3fc6810d8 namespace: kserve-ci-e2e-test resourceVersion: "17342" uid: 9bbf0b12-cb9d-476d-b465-efbcf8a72753 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-766f3 namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: b3a81fd3-944f-4970-87be-ba7c4ef8521d kind: Event lastTimestamp: "2026-07-01T17:30:47Z" message: Scaled up replica set sequence-graph-766f3-f9786597b from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:47Z" name: sequence-graph-766f3.18be3ab3f95beaf4 namespace: kserve-ci-e2e-test resourceVersion: "17337" uid: 3f623605-8500-43c3-9af8-7f7e0804802c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-766f3 namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: dbc525d2-12f3-41e7-8860-f4497d99958f kind: Event lastTimestamp: "2026-07-01T17:30:47Z" message: InferenceGraph [sequence-graph-766f3] is Ready metadata: creationTimestamp: "2026-07-01T17:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:47Z" name: sequence-graph-766f3.18be3ab4044789d0 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: 6f2628d1-03bd-4543-a328-d990363230f7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-766f3 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 139667c6-12e2-44a4-a4de-17e6d74fd71d kind: Event lastTimestamp: "2026-07-01T17:31:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:17Z" name: sequence-graph-766f3.18be3ab7789b171b namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: e55692bb-1e78-4348-8874-6e16d9a77864 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-766f3 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 139667c6-12e2-44a4-a4de-17e6d74fd71d kind: Event lastTimestamp: "2026-07-01T17:31:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:17Z" name: sequence-graph-766f3.18be3ab7791473d0 namespace: kserve-ci-e2e-test resourceVersion: "17581" uid: a5b7991d-7aad-43e3-9d17-9609f8eb2b83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:41:22.250552Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8b57f-6c5d78755-f6zxd namespace: kserve-ci-e2e-test resourceVersion: "22687" uid: b50d7a8a-f0ca-464f-8956-cd51ff1dd989 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8b57f-6c5d78755-f6zxd to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:41:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:41:22Z" name: sequence-graph-8b57f-6c5d78755-f6zxd.18be3b47d8c15713 namespace: kserve-ci-e2e-test resourceVersion: "22692" uid: eade4b2c-2117-4e41-b71b-1044e1937aeb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:22Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8b57f-6c5d78755-f6zxd namespace: kserve-ci-e2e-test resourceVersion: "22690" uid: b50d7a8a-f0ca-464f-8956-cd51ff1dd989 kind: Event lastTimestamp: "2026-07-01T17:41:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-8b57f-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:41:22Z" name: sequence-graph-8b57f-6c5d78755-f6zxd.18be3b47e351e089 namespace: kserve-ci-e2e-test resourceVersion: "22703" uid: 0d91f41b-e2a6-4ca1-bca4-36b1e06da951 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:23Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8b57f-6c5d78755-f6zxd namespace: kserve-ci-e2e-test resourceVersion: "22696" uid: b50d7a8a-f0ca-464f-8956-cd51ff1dd989 kind: Event lastTimestamp: "2026-07-01T17:41:23Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:41:23Z" name: sequence-graph-8b57f-6c5d78755-f6zxd.18be3b48166557b9 namespace: kserve-ci-e2e-test resourceVersion: "22712" uid: 4c1af975-5cbf-4683-98e4-414736610bce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b57f} kind: Pod name: sequence-graph-8b57f-6c5d78755-f6zxd namespace: kserve-ci-e2e-test resourceVersion: "22690" uid: b50d7a8a-f0ca-464f-8956-cd51ff1dd989 kind: Event lastTimestamp: "2026-07-01T17:41:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:41:23Z" name: sequence-graph-8b57f-6c5d78755-f6zxd.18be3b4817af31af namespace: kserve-ci-e2e-test resourceVersion: "22714" uid: 9ad94111-943c-4fa2-b361-068d5a296225 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b57f} kind: Pod name: sequence-graph-8b57f-6c5d78755-f6zxd namespace: kserve-ci-e2e-test resourceVersion: "22690" uid: b50d7a8a-f0ca-464f-8956-cd51ff1dd989 kind: Event lastTimestamp: "2026-07-01T17:41:23Z" message: 'Created container: sequence-graph-8b57f' metadata: creationTimestamp: "2026-07-01T17:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:41:23Z" name: sequence-graph-8b57f-6c5d78755-f6zxd.18be3b481bb93811 namespace: kserve-ci-e2e-test resourceVersion: "22715" uid: c803723d-bdc8-44e0-b837-aa482a6d32d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b57f} kind: Pod name: sequence-graph-8b57f-6c5d78755-f6zxd namespace: kserve-ci-e2e-test resourceVersion: "22690" uid: b50d7a8a-f0ca-464f-8956-cd51ff1dd989 kind: Event lastTimestamp: "2026-07-01T17:41:23Z" message: Started container sequence-graph-8b57f metadata: creationTimestamp: "2026-07-01T17:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:41:23Z" name: sequence-graph-8b57f-6c5d78755-f6zxd.18be3b481c346814 namespace: kserve-ci-e2e-test resourceVersion: "22717" uid: 9904b163-bf0a-486f-8a7c-3d137ccde2c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b57f} kind: Pod name: sequence-graph-8b57f-6c5d78755-f6zxd namespace: kserve-ci-e2e-test resourceVersion: "22690" uid: b50d7a8a-f0ca-464f-8956-cd51ff1dd989 kind: Event lastTimestamp: "2026-07-01T17:49:36Z" message: Stopping container sequence-graph-8b57f metadata: creationTimestamp: "2026-07-01T17:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:36Z" name: sequence-graph-8b57f-6c5d78755-f6zxd.18be3bbb08046844 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: d6339e3f-19ca-4df4-8c24-d33ab180903b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8b57f} kind: Pod name: sequence-graph-8b57f-6c5d78755-f6zxd namespace: kserve-ci-e2e-test resourceVersion: "22690" uid: b50d7a8a-f0ca-464f-8956-cd51ff1dd989 kind: Event lastTimestamp: "2026-07-01T17:50:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:05Z" name: sequence-graph-8b57f-6c5d78755-f6zxd.18be3bbbbf81cd98 namespace: kserve-ci-e2e-test resourceVersion: "27148" uid: 37501164-302d-4492-a500-9387bd487f66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8b57f-6c5d78755 namespace: kserve-ci-e2e-test resourceVersion: "22684" uid: c019669a-b705-4afd-807a-0dad1610aea0 kind: Event lastTimestamp: "2026-07-01T17:41:22Z" message: 'Created pod: sequence-graph-8b57f-6c5d78755-f6zxd' metadata: creationTimestamp: "2026-07-01T17:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:22Z" name: sequence-graph-8b57f-6c5d78755.18be3b47d832f90b namespace: kserve-ci-e2e-test resourceVersion: "22689" uid: 2ea795fc-63cd-4850-b254-4613c2f0da0b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8b57f namespace: kserve-ci-e2e-test resourceVersion: "22682" uid: fa6b01f8-7ec4-4823-bb1d-4a9b0c54417d kind: Event lastTimestamp: "2026-07-01T17:41:22Z" message: Scaled up replica set sequence-graph-8b57f-6c5d78755 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:22Z" name: sequence-graph-8b57f.18be3b47d74b0715 namespace: kserve-ci-e2e-test resourceVersion: "22686" uid: 6ede5bb9-bbd7-4dc9-9801-d0b8ff29202f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8b57f namespace: kserve-ci-e2e-test resourceVersion: "22698" uid: e6e79e33-9453-4f3a-b0d5-2e7faab0995c kind: Event lastTimestamp: "2026-07-01T17:41:22Z" message: InferenceGraph [sequence-graph-8b57f] is Ready metadata: creationTimestamp: "2026-07-01T17:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:41:22Z" name: sequence-graph-8b57f.18be3b47db22482e namespace: kserve-ci-e2e-test resourceVersion: "22700" uid: 85548ddd-331c-4dc9-b032-431fb825c18f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:41:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8b57f namespace: kserve-ci-e2e-test resourceVersion: "22683" uid: 4e19be93-4cb0-4113-a549-d8ec902d5f2e kind: Event lastTimestamp: "2026-07-01T17:41:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:52Z" name: sequence-graph-8b57f.18be3b4b561fba96 namespace: kserve-ci-e2e-test resourceVersion: "22932" uid: 3d7abd72-794b-4d2f-96fd-807fae01afcf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:41:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8b57f namespace: kserve-ci-e2e-test resourceVersion: "22683" uid: 4e19be93-4cb0-4113-a549-d8ec902d5f2e kind: Event lastTimestamp: "2026-07-01T17:41:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:41:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:52Z" name: sequence-graph-8b57f.18be3b4b56810cdf namespace: kserve-ci-e2e-test resourceVersion: "22933" uid: e7ac8c2f-8229-4adc-9b9a-61e9f60e37a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:40:01.970113Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-98f1c-788bc79767-kkl6d namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 26e5d5c9-ef69-4bef-9383-df0e1e3de06a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-98f1c-788bc79767-kkl6d to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:40:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:40:01Z" name: sequence-graph-98f1c-788bc79767-kkl6d.18be3b3527ab0099 namespace: kserve-ci-e2e-test resourceVersion: "21802" uid: 71ec041e-7e72-4a15-82dd-fe060456a68f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-98f1c-788bc79767-kkl6d namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 26e5d5c9-ef69-4bef-9383-df0e1e3de06a kind: Event lastTimestamp: "2026-07-01T17:40:02Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:40:02Z" name: sequence-graph-98f1c-788bc79767-kkl6d.18be3b354127782f namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: 0220663f-c685-47f7-961e-8dcfe9b3e2e8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-98f1c} kind: Pod name: sequence-graph-98f1c-788bc79767-kkl6d namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 26e5d5c9-ef69-4bef-9383-df0e1e3de06a kind: Event lastTimestamp: "2026-07-01T17:40:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:02Z" name: sequence-graph-98f1c-788bc79767-kkl6d.18be3b35426c0ede namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: 224da907-b835-4b1f-b11e-8ba797200032 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-98f1c} kind: Pod name: sequence-graph-98f1c-788bc79767-kkl6d namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 26e5d5c9-ef69-4bef-9383-df0e1e3de06a kind: Event lastTimestamp: "2026-07-01T17:40:02Z" message: 'Created container: sequence-graph-98f1c' metadata: creationTimestamp: "2026-07-01T17:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:02Z" name: sequence-graph-98f1c-788bc79767-kkl6d.18be3b35452af3ce namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: c7109722-780e-41e5-a3af-09d23effe3ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-98f1c} kind: Pod name: sequence-graph-98f1c-788bc79767-kkl6d namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 26e5d5c9-ef69-4bef-9383-df0e1e3de06a kind: Event lastTimestamp: "2026-07-01T17:40:02Z" message: Started container sequence-graph-98f1c metadata: creationTimestamp: "2026-07-01T17:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:02Z" name: sequence-graph-98f1c-788bc79767-kkl6d.18be3b3545a17382 namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: 7c6f7525-9230-40e9-b12e-b7060431ec67 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-98f1c} kind: Pod name: sequence-graph-98f1c-788bc79767-kkl6d namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 26e5d5c9-ef69-4bef-9383-df0e1e3de06a kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: Stopping container sequence-graph-98f1c metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:12Z" name: sequence-graph-98f1c-788bc79767-kkl6d.18be3b3780daf4b0 namespace: kserve-ci-e2e-test resourceVersion: "21934" uid: c13130e8-38b6-4f9e-888c-af8cd68e8e19 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-98f1c} kind: Pod name: sequence-graph-98f1c-788bc79767-kkl6d namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 26e5d5c9-ef69-4bef-9383-df0e1e3de06a kind: Event lastTimestamp: "2026-07-01T17:40:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:38Z" name: sequence-graph-98f1c-788bc79767-kkl6d.18be3b37e39b8b94 namespace: kserve-ci-e2e-test resourceVersion: "22331" uid: a5966bdf-ea16-4a9f-9924-b861b998c619 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-98f1c-788bc79767 namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 8cf688e4-2dab-4ea6-9aba-a1173f1b7207 kind: Event lastTimestamp: "2026-07-01T17:40:01Z" message: 'Created pod: sequence-graph-98f1c-788bc79767-kkl6d' metadata: creationTimestamp: "2026-07-01T17:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:01Z" name: sequence-graph-98f1c-788bc79767.18be3b35273e6faf namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: 01742baf-fa7b-4350-90da-9d35f591a44a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-98f1c namespace: kserve-ci-e2e-test resourceVersion: "21792" uid: de8970f8-ef35-482d-9b42-719de9a4f441 kind: Event lastTimestamp: "2026-07-01T17:40:01Z" message: Scaled up replica set sequence-graph-98f1c-788bc79767 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:01Z" name: sequence-graph-98f1c.18be3b352681208a namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: 56bb304d-abf1-4dbf-989e-60e0e53196a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-98f1c namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: ecf1ab95-d216-4098-b01e-9951a03eaec5 kind: Event lastTimestamp: "2026-07-01T17:40:02Z" message: InferenceGraph [sequence-graph-98f1c] is Ready metadata: creationTimestamp: "2026-07-01T17:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:02Z" name: sequence-graph-98f1c.18be3b352c055ae1 namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: dd905370-6c18-47e7-b88b-03e603c0fb3f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:49:41.460379Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-10517-86976cfcdc-6xlxr namespace: kserve-ci-e2e-test resourceVersion: "26785" uid: 7abc0508-1bab-449c-985f-d87bae73b814 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-10517-86976cfcdc-6xlxr to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517-86976cfcdc-6xlxr.18be3bbc13fac135 namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: 714dd127-649f-4f2e-8f3b-2bcc23f7bd4f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-10517-86976cfcdc-6xlxr namespace: kserve-ci-e2e-test resourceVersion: "26793" uid: 7abc0508-1bab-449c-985f-d87bae73b814 kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517-86976cfcdc-6xlxr.18be3bbc2d8ed97f namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 2308e21a-72d8-4861-8c00-877800052907 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-10517} kind: Pod name: splitter-graph-10517-86976cfcdc-6xlxr namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 7abc0508-1bab-449c-985f-d87bae73b814 kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517-86976cfcdc-6xlxr.18be3bbc2edcdd09 namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: e2c0e946-6981-46d9-8adc-34733dd70273 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-10517} kind: Pod name: splitter-graph-10517-86976cfcdc-6xlxr namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 7abc0508-1bab-449c-985f-d87bae73b814 kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: 'Created container: splitter-graph-10517' metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517-86976cfcdc-6xlxr.18be3bbc31d1acb2 namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: c00cc9dd-ff5c-4e58-9671-50a34e863582 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-10517} kind: Pod name: splitter-graph-10517-86976cfcdc-6xlxr namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 7abc0508-1bab-449c-985f-d87bae73b814 kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: Started container splitter-graph-10517 metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517-86976cfcdc-6xlxr.18be3bbc326098ef namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: ebfcb97a-c77d-4b6d-9a1e-56fc497ddd87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-10517} kind: Pod name: splitter-graph-10517-86976cfcdc-6xlxr namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 7abc0508-1bab-449c-985f-d87bae73b814 kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: Stopping container splitter-graph-10517 metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:51Z" name: splitter-graph-10517-86976cfcdc-6xlxr.18be3bbe6cbd5b45 namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: 73334155-7682-4628-9f3f-817a30e339ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:49:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-10517} kind: Pod name: splitter-graph-10517-86976cfcdc-6xlxr namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 7abc0508-1bab-449c-985f-d87bae73b814 kind: Event lastTimestamp: "2026-07-01T17:50:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:18Z" name: splitter-graph-10517-86976cfcdc-6xlxr.18be3bbec5d67a6a namespace: kserve-ci-e2e-test resourceVersion: "27269" uid: f07b0544-24cd-409c-b9a0-42671bf53646 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-10517-86976cfcdc namespace: kserve-ci-e2e-test resourceVersion: "26782" uid: fea15e11-c759-4cd3-b2f6-76d3c9e46fbf kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: 'Created pod: splitter-graph-10517-86976cfcdc-6xlxr' metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517-86976cfcdc.18be3bbc1367295e namespace: kserve-ci-e2e-test resourceVersion: "26786" uid: 15286df6-6fae-4239-b811-a24df597a771 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-10517 namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: 5f0ac73b-e232-456b-9673-7354d9f1583e kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: Scaled up replica set splitter-graph-10517-86976cfcdc from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517.18be3bbc12ace90e namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: cddb802e-786a-42f7-a22d-ad72dd405380 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-10517 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: 17a4f607-f911-47db-905e-5ebb6458a3ae kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: InferenceGraph [splitter-graph-10517] is Ready metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517.18be3bbc1810e926 namespace: kserve-ci-e2e-test resourceVersion: "26798" uid: d3b7b040-e67e-4e6b-b038-3f4309222217 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-10517 namespace: kserve-ci-e2e-test resourceVersion: "26772" uid: 17a4f607-f911-47db-905e-5ebb6458a3ae kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: 'Failed to update status for InferenceGraph "splitter-graph-10517": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-10517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517.18be3bbc1a7120fb namespace: kserve-ci-e2e-test resourceVersion: "26802" uid: e6c8f64e-d004-429d-b4b3-5745d949bcb8 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-10517 namespace: kserve-ci-e2e-test resourceVersion: "26772" uid: 17a4f607-f911-47db-905e-5ebb6458a3ae kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-10517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:41Z" name: splitter-graph-10517.18be3bbc1a716286 namespace: kserve-ci-e2e-test resourceVersion: "26803" uid: 524f2ac6-3fd9-4683-abff-948f40cabe99 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:50:51.741210Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-42b28-5f7d665cfb-9t2zj namespace: kserve-ci-e2e-test resourceVersion: "27608" uid: f9b45c3b-ae08-4355-8930-ecdfd03190d4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-42b28-5f7d665cfb-9t2zj to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:50:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:50:51Z" name: splitter-graph-42b28-5f7d665cfb-9t2zj.18be3bcc710b1c1f namespace: kserve-ci-e2e-test resourceVersion: "27613" uid: 4fa8e678-7c43-4f4a-8120-71f06d76427d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:52Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-42b28-5f7d665cfb-9t2zj namespace: kserve-ci-e2e-test resourceVersion: "27617" uid: f9b45c3b-ae08-4355-8930-ecdfd03190d4 kind: Event lastTimestamp: "2026-07-01T17:50:52Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:50:52Z" name: splitter-graph-42b28-5f7d665cfb-9t2zj.18be3bcc8a9e7956 namespace: kserve-ci-e2e-test resourceVersion: "27634" uid: 0f61513a-ef9d-4eea-b66c-b295a991d881 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-42b28} kind: Pod name: splitter-graph-42b28-5f7d665cfb-9t2zj namespace: kserve-ci-e2e-test resourceVersion: "27610" uid: f9b45c3b-ae08-4355-8930-ecdfd03190d4 kind: Event lastTimestamp: "2026-07-01T17:50:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:52Z" name: splitter-graph-42b28-5f7d665cfb-9t2zj.18be3bcc8be66030 namespace: kserve-ci-e2e-test resourceVersion: "27639" uid: 5842b752-4b70-4178-8362-bba47449aa36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-42b28} kind: Pod name: splitter-graph-42b28-5f7d665cfb-9t2zj namespace: kserve-ci-e2e-test resourceVersion: "27610" uid: f9b45c3b-ae08-4355-8930-ecdfd03190d4 kind: Event lastTimestamp: "2026-07-01T17:50:52Z" message: 'Created container: splitter-graph-42b28' metadata: creationTimestamp: "2026-07-01T17:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:52Z" name: splitter-graph-42b28-5f7d665cfb-9t2zj.18be3bcc8ee81fa1 namespace: kserve-ci-e2e-test resourceVersion: "27641" uid: 9b94da40-c8ee-4a43-bfca-c85e0dc13214 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-42b28} kind: Pod name: splitter-graph-42b28-5f7d665cfb-9t2zj namespace: kserve-ci-e2e-test resourceVersion: "27610" uid: f9b45c3b-ae08-4355-8930-ecdfd03190d4 kind: Event lastTimestamp: "2026-07-01T17:50:52Z" message: Started container splitter-graph-42b28 metadata: creationTimestamp: "2026-07-01T17:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:52Z" name: splitter-graph-42b28-5f7d665cfb-9t2zj.18be3bcc8f60f087 namespace: kserve-ci-e2e-test resourceVersion: "27642" uid: aa87eff0-fc81-4ffc-8c78-557a1666cd9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-42b28} kind: Pod name: splitter-graph-42b28-5f7d665cfb-9t2zj namespace: kserve-ci-e2e-test resourceVersion: "27610" uid: f9b45c3b-ae08-4355-8930-ecdfd03190d4 kind: Event lastTimestamp: "2026-07-01T17:59:06Z" message: Stopping container splitter-graph-42b28 metadata: creationTimestamp: "2026-07-01T17:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:06Z" name: splitter-graph-42b28-5f7d665cfb-9t2zj.18be3c3f928f1cd6 namespace: kserve-ci-e2e-test resourceVersion: "31079" uid: ddb84c97-f2c4-4970-9a01-c84a0d5f1788 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:59:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-42b28} kind: Pod name: splitter-graph-42b28-5f7d665cfb-9t2zj namespace: kserve-ci-e2e-test resourceVersion: "27610" uid: f9b45c3b-ae08-4355-8930-ecdfd03190d4 kind: Event lastTimestamp: "2026-07-01T17:59:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:59:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:33Z" name: splitter-graph-42b28-5f7d665cfb-9t2zj.18be3c400f97e2b6 namespace: kserve-ci-e2e-test resourceVersion: "31318" uid: 0c762d8e-16a7-4f6b-a76e-d763d371b041 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-42b28-5f7d665cfb namespace: kserve-ci-e2e-test resourceVersion: "27604" uid: 9badfb02-18f2-4e53-b281-6294a210245f kind: Event lastTimestamp: "2026-07-01T17:50:51Z" message: 'Created pod: splitter-graph-42b28-5f7d665cfb-9t2zj' metadata: creationTimestamp: "2026-07-01T17:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:51Z" name: splitter-graph-42b28-5f7d665cfb.18be3bcc70981f41 namespace: kserve-ci-e2e-test resourceVersion: "27609" uid: a066729b-2331-49eb-9c14-c9f60674fee7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-42b28 namespace: kserve-ci-e2e-test resourceVersion: "27601" uid: 9178e161-8ad6-49bd-83fe-dd6ed6b38da0 kind: Event lastTimestamp: "2026-07-01T17:50:51Z" message: Scaled up replica set splitter-graph-42b28-5f7d665cfb from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:51Z" name: splitter-graph-42b28.18be3bcc6ffc0117 namespace: kserve-ci-e2e-test resourceVersion: "27606" uid: cfbce41e-b571-4a68-ab36-751d1b663873 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-42b28 namespace: kserve-ci-e2e-test resourceVersion: "27620" uid: 7046ea45-80c4-4b5e-968a-6b366e48744a kind: Event lastTimestamp: "2026-07-01T17:50:51Z" message: InferenceGraph [splitter-graph-42b28] is Ready metadata: creationTimestamp: "2026-07-01T17:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:51Z" name: splitter-graph-42b28.18be3bcc7503742e namespace: kserve-ci-e2e-test resourceVersion: "27623" uid: 9ddca2ea-c7d3-47b1-bf08-ed9d06d15507 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:51:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-42b28 namespace: kserve-ci-e2e-test resourceVersion: "27603" uid: f105038d-98cd-4e79-b797-3e3d73b7a8e3 kind: Event lastTimestamp: "2026-07-01T17:51:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:51:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:21Z" name: splitter-graph-42b28.18be3bcfee93a6d7 namespace: kserve-ci-e2e-test resourceVersion: "27866" uid: 18b913ed-b7dc-4cb4-956f-102daba551c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:51:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-42b28 namespace: kserve-ci-e2e-test resourceVersion: "27603" uid: f105038d-98cd-4e79-b797-3e3d73b7a8e3 kind: Event lastTimestamp: "2026-07-01T17:51:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:51:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:21Z" name: splitter-graph-42b28.18be3bcfeed7288b namespace: kserve-ci-e2e-test resourceVersion: "27867" uid: 7a04e72e-e1ae-4407-afdc-383dbecd67f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:48:41.435403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26054" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-10517-predictor-54485f9447-6xsj6 to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bae1a3658f9 namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: 30a64d85-3130-4b65-a27a-986666709c6f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26076" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bae34df9eeb namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: f3b88ed8-48d5-4543-9017-56b75bc37dfd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bae3644bc4a namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 666062a3-6e30-494c-a111-452534e6c85d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bae39e611fc namespace: kserve-ci-e2e-test resourceVersion: "26130" uid: be1e2256-c90d-4f8f-9330-35f188345b5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bae3a5aa0d0 namespace: kserve-ci-e2e-test resourceVersion: "26131" uid: 226f660f-2940-4743-85fe-763f671b9e09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bae3a658e6b namespace: kserve-ci-e2e-test resourceVersion: "26132" uid: 0d32887d-2339-4e43-8ce1-c430ac1feef8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:48:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:42Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bae3d55ec48 namespace: kserve-ci-e2e-test resourceVersion: "26135" uid: 2f1895fe-9c42-4a9d-b7b6-940c61c1422e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:48:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:42Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bae3e1c006e namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 795ffbd5-21db-4734-abc4-609589b905bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:48:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:49:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:19Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bae79ca254e namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: a3208a24-4240-498f-99ba-79e30240e4a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:51Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bbe72d37c43 namespace: kserve-ci-e2e-test resourceVersion: "26941" uid: 02b239e3-f264-40ea-9744-68c348f4069a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:51Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bbe72d7d043 namespace: kserve-ci-e2e-test resourceVersion: "26945" uid: 5da7c1a6-fb87-4465-ac8e-0b31a4ebad64 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-10517-predictor-54485f9447-6xsj6 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 4d77ffc0-8d6c-48f1-90ae-abe293506504 kind: Event lastTimestamp: "2026-07-01T17:49:54Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:54Z" name: success-200-isvc-10517-predictor-54485f9447-6xsj6.18be3bbf01a06aa5 namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: 55f55663-d379-4ef0-ae4a-f367c8dd0cf8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-10517-predictor-54485f9447 namespace: kserve-ci-e2e-test resourceVersion: "26035" uid: 74b66eaa-1590-4ee7-86ed-53c9c56d9ee8 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: 'Created pod: success-200-isvc-10517-predictor-54485f9447-6xsj6' metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517-predictor-54485f9447.18be3bae19a07ef4 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 9819b377-b2b2-4ae1-aad0-906e21f45819 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: fc928d5c-8720-478f-b8cf-e2a1a41fc511 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Scaled up replica set success-200-isvc-10517-predictor-54485f9447 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517-predictor.18be3bae15fe1a2f namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: c0b4f1e3-2608-401d-861b-1f2b0e4cbf3d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: 857b6323-b6ca-492c-9c9b-a3848efef433 kind: Event lastTimestamp: "2026-07-01T17:49:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:26Z" name: success-200-isvc-10517-predictor.18be3bb194517020 namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 0e4ef6ac-6359-4c8f-ae35-1232e9b8ceaa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: 857b6323-b6ca-492c-9c9b-a3848efef433 kind: Event lastTimestamp: "2026-07-01T17:49:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:26Z" name: success-200-isvc-10517-predictor.18be3bb194b67a82 namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: a7adba71-1d63-49b1-aed8-bcdb4ff7cf03 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: 857b6323-b6ca-492c-9c9b-a3848efef433 kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:41Z" name: success-200-isvc-10517-predictor.18be3bbc10d4da38 namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: 03159970-3dae-4ee8-8d50-39ac87a9a0e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-10517-predictor namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: 857b6323-b6ca-492c-9c9b-a3848efef433 kind: Event lastTimestamp: "2026-07-01T17:49:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:41Z" name: success-200-isvc-10517-predictor.18be3bbc113992af namespace: kserve-ci-e2e-test resourceVersion: "26777" uid: a297de13-2514-4102-8c23-f4204e620c8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-10517 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 420541fc-aa0e-4eba-acd3-407aaa4e0eee kind: Event lastTimestamp: "2026-07-01T17:49:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:29Z" name: success-200-isvc-10517.18be3bae1601b038 namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: c57d0a7a-9d27-40d2-967c-d4f7d9281ed3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-10517 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 420541fc-aa0e-4eba-acd3-407aaa4e0eee kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-10517": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-10517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517.18be3bae16746d1b namespace: kserve-ci-e2e-test resourceVersion: "26039" uid: 6e3ac269-89f6-406d-ad65-07e19cd8726c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-10517 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 420541fc-aa0e-4eba-acd3-407aaa4e0eee kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-10517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-10517.18be3bae1674ab32 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: 0044d41d-a516-4382-ad13-7831b2c3b3aa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-10517 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 420541fc-aa0e-4eba-acd3-407aaa4e0eee kind: Event lastTimestamp: "2026-07-01T17:49:29Z" message: InferenceService [success-200-isvc-10517] is Ready metadata: creationTimestamp: "2026-07-01T17:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:29Z" name: success-200-isvc-10517.18be3bb934a75887 namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: 65357fde-079d-4c5f-9f4d-bf2c7e486042 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:49:51.677597Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26949" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-42b28-predictor-596b5858f9-q5wps to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:49:51Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3bbe74f92a80 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: e3e008d4-cc03-4455-af51-932a492fc672 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26960" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:49:52Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3bbe8ee0f894 namespace: kserve-ci-e2e-test resourceVersion: "27019" uid: c355fc0f-f5a2-4fcb-a141-fcdee42f72b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3bbe902490e2 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 6c0872b9-8a7e-4e25-880d-c5aa2a0b7251 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3bbe933610c2 namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: d3e3b9bf-c201-46b3-864b-9b50c6ec51dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3bbe93a8f38f namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: 667efeb9-67bd-44cd-b0b8-b8eea0e83d69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3bbe93b2c691 namespace: kserve-ci-e2e-test resourceVersion: "27027" uid: 7cb611a2-79f2-4861-ab8f-dea040da1adb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3bbe96ff487f namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 4a24c12e-4ee2-4dc6-bde9-e3a2f5ba27a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:52Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3bbe976c436c namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: 1c36e296-fa54-489c-9568-2f2c4f73d650 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:49:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:59:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:09Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3bbed9c64d0e namespace: kserve-ci-e2e-test resourceVersion: "31139" uid: df5918fd-6353-4230-99b7-37244b3bdc8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:59:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:06Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3c3f9adf5dfe namespace: kserve-ci-e2e-test resourceVersion: "31095" uid: b21e53bf-e9e2-4695-b407-6b977cc420f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:59:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:06Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3c3f9adfea34 namespace: kserve-ci-e2e-test resourceVersion: "31097" uid: adb7a0b7-c1c0-4fb6-96b6-f67e21e84687 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42b28-predictor-596b5858f9-q5wps namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 17084014-45f0-4920-bfc3-99d6187448ff kind: Event lastTimestamp: "2026-07-01T17:59:09Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:59:09Z" name: success-200-isvc-42b28-predictor-596b5858f9-q5wps.18be3c404e25c821 namespace: kserve-ci-e2e-test resourceVersion: "31138" uid: 2e056dec-4d5f-4210-b482-ef4e8ff8e2d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-42b28-predictor-596b5858f9 namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: d2cb964c-4fc5-4844-99b7-3acf3959784d kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: 'Created pod: success-200-isvc-42b28-predictor-596b5858f9-q5wps' metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:51Z" name: success-200-isvc-42b28-predictor-596b5858f9.18be3bbe748f8467 namespace: kserve-ci-e2e-test resourceVersion: "26951" uid: 3df3ab52-5fd9-4e78-98b3-48b1a49197b3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "26934" uid: 8ae26f64-4ecc-4b8f-abc1-6d578ce1a21f kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: Scaled up replica set success-200-isvc-42b28-predictor-596b5858f9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:51Z" name: success-200-isvc-42b28-predictor.18be3bbe72ca6aa0 namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: fce32bed-6287-4aac-bc06-8266923c8f01 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "26937" uid: cbe41257-6aa2-4ba9-82fb-8bf0f2d510e7 kind: Event lastTimestamp: "2026-07-01T17:50:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:21Z" name: success-200-isvc-42b28-predictor.18be3bc1f1f738f3 namespace: kserve-ci-e2e-test resourceVersion: "27291" uid: 2a11c4b5-bc7d-4020-8c3f-fc98ff8644d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "26937" uid: cbe41257-6aa2-4ba9-82fb-8bf0f2d510e7 kind: Event lastTimestamp: "2026-07-01T17:50:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:21Z" name: success-200-isvc-42b28-predictor.18be3bc1f2587d4a namespace: kserve-ci-e2e-test resourceVersion: "27292" uid: 8a8177b6-b412-4b33-9970-4edae6ed7993 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "27165" uid: cbe41257-6aa2-4ba9-82fb-8bf0f2d510e7 kind: Event lastTimestamp: "2026-07-01T17:51:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:06Z" name: success-200-isvc-42b28-predictor.18be3bc8f03c5d02 namespace: kserve-ci-e2e-test resourceVersion: "27754" uid: 180004c4-17e3-469c-9ce4-efc321b9bac5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42b28-predictor namespace: kserve-ci-e2e-test resourceVersion: "27165" uid: cbe41257-6aa2-4ba9-82fb-8bf0f2d510e7 kind: Event lastTimestamp: "2026-07-01T17:51:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:06Z" name: success-200-isvc-42b28-predictor.18be3bc8f0a34cd5 namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 33908bea-0387-4f3d-83cf-6bf763fc333e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42b28 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 66e26438-f45c-4de3-b296-bff10382fa03 kind: Event lastTimestamp: "2026-07-01T17:50:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:39Z" name: success-200-isvc-42b28.18be3bbe731b3750 namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: 498bc312-9b9d-4808-af40-dbbe7a122d44 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42b28 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 66e26438-f45c-4de3-b296-bff10382fa03 kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-42b28": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-42b28": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:51Z" name: success-200-isvc-42b28.18be3bbe738ae725 namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 26fdf583-264b-4433-8493-e8681347baa5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42b28 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 66e26438-f45c-4de3-b296-bff10382fa03 kind: Event lastTimestamp: "2026-07-01T17:49:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-42b28": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:51Z" name: success-200-isvc-42b28.18be3bbe738b560c namespace: kserve-ci-e2e-test resourceVersion: "26948" uid: b3a400b9-c41e-45d4-880c-c7c786d437c7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42b28 namespace: kserve-ci-e2e-test resourceVersion: "27502" uid: 66e26438-f45c-4de3-b296-bff10382fa03 kind: Event lastTimestamp: "2026-07-01T17:50:39Z" message: InferenceService [success-200-isvc-42b28] is Ready metadata: creationTimestamp: "2026-07-01T17:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:39Z" name: success-200-isvc-42b28.18be3bc995580f6a namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: 24bd4c83-7236-4ba5-b70d-86496c7c2547 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:38:16.303439Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:38:16Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1c8d719b62 namespace: kserve-ci-e2e-test resourceVersion: "20500" uid: f16cf432-3886-4ae5-bf9f-f7532c2077d8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-43ae3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:16Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1c991761f1 namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: 7b5c25b6-0562-4602-9c44-09a2962c5e31 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:38:17Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1ccae24d97 namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: cff2e89b-e411-40b0-a113-924b7dfe2e43 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1ccd0f83d2 namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: b3a32e13-dfcf-4ca5-a4ce-44bd5ba8e106 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1cd0411bb7 namespace: kserve-ci-e2e-test resourceVersion: "20573" uid: 6a5d5924-6dbe-4702-ab57-047a616de92f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1cd0ab56e6 namespace: kserve-ci-e2e-test resourceVersion: "20574" uid: e35714a3-09ff-4f4f-b1d7-38f8718630d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1cd0b3b0b1 namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 3c5821ff-e398-43a1-817f-ab5553c63971 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1cd39326b9 namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: b9f8c181-abaf-4d4a-bf6c-20ba9aba2b84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:38:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:38:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:17Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1cd3ffe094 namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 6fc60830-8030-4b67-bb35-51ff9b9455c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:38:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:55Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b1d397e3f6d namespace: kserve-ci-e2e-test resourceVersion: "20894" uid: 2d4bda51-a28d-4599-a882-20982e490104 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b2ce0dbc081 namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 4d40d5fc-60d4-47d7-b553-31e749d322b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 392a47f8-ca87-465e-9908-750f73ac321c kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9.18be3b2ce0df3f69 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: daceb371-09a9-4ec4-87d9-ae09c3eb9e04 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-43ae3-predictor-555ccc9b67 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: 098384f7-8b3c-4b6a-b9f2-2f21ffb667f0 kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: 'Created pod: success-200-isvc-43ae3-predictor-555ccc9b67-xb6r9' metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:38:16Z" name: success-200-isvc-43ae3-predictor-555ccc9b67.18be3b1c8cefd828 namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: bfe4f05f-436f-4e03-bbe5-a319f8e7ef82 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: e03b5f64-880f-4111-9a11-74f8eb0e4029 kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: Scaled up replica set success-200-isvc-43ae3-predictor-555ccc9b67 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:38:16Z" name: success-200-isvc-43ae3-predictor.18be3b1c89910ceb namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: dfd1e446-fb88-4482-8f28-44357add6100 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:38:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 67653b73-e1dc-4823-ba0f-4404f0608c6b kind: Event lastTimestamp: "2026-07-01T17:38:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:38:46Z" name: success-200-isvc-43ae3-predictor.18be3b20082ed9af namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 8153530f-6474-4e00-b075-fb61ff5aa58e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:38:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 67653b73-e1dc-4823-ba0f-4404f0608c6b kind: Event lastTimestamp: "2026-07-01T17:38:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:38:46Z" name: success-200-isvc-43ae3-predictor.18be3b20089b876e namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: fd1a11e8-b9e4-4887-9a11-491228b1ec89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 67653b73-e1dc-4823-ba0f-4404f0608c6b kind: Event lastTimestamp: "2026-07-01T17:39:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:16Z" name: success-200-isvc-43ae3-predictor.18be3b2706a52232 namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: f727bfb4-22c5-4fcf-83d1-5078bc7146da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43ae3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 67653b73-e1dc-4823-ba0f-4404f0608c6b kind: Event lastTimestamp: "2026-07-01T17:39:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:16Z" name: success-200-isvc-43ae3-predictor.18be3b2706f1b2de namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: fa6fb873-e85f-4135-89b0-1ddc5f8a80d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43ae3 namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: 616ff152-30ba-4ae4-8c32-d242f11e3eeb kind: Event lastTimestamp: "2026-07-01T17:39:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:05Z" name: success-200-isvc-43ae3.18be3b1c899a7108 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 15137367-8551-442d-ab32-11957c3a5736 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43ae3 namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: 616ff152-30ba-4ae4-8c32-d242f11e3eeb kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-43ae3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-43ae3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:38:16Z" name: success-200-isvc-43ae3.18be3b1c8a107c3f namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 2575f52f-8d99-421f-90fe-c42c95edf076 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43ae3 namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: 616ff152-30ba-4ae4-8c32-d242f11e3eeb kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-43ae3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:38:16Z" name: success-200-isvc-43ae3.18be3b1c8a10baf3 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 7ec0965c-ffba-4716-89dc-a9031a4cc1ae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43ae3 namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: 616ff152-30ba-4ae4-8c32-d242f11e3eeb kind: Event lastTimestamp: "2026-07-01T17:39:05Z" message: InferenceService [success-200-isvc-43ae3] is Ready metadata: creationTimestamp: "2026-07-01T17:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:05Z" name: success-200-isvc-43ae3.18be3b27f5971f20 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 81d01ee6-817d-42d0-9fa3-071631aee73b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:49:37.103720Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26641" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4822a-predictor-59f89c648f-gb6hs to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3bbb104d791d namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: e007e9ac-1f13-430b-aaa8-a14a3dd55be0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26658" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3bbb2a476562 namespace: kserve-ci-e2e-test resourceVersion: "26704" uid: 22cb6993-bfcb-44ec-ae05-3731263120d4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3bbb2b900e80 namespace: kserve-ci-e2e-test resourceVersion: "26706" uid: d15f465d-29e9-4948-a72d-139adf1027b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3bbb2eb38d64 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 911555b0-24ab-4482-89ce-99887f15f118 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3bbb2f2636b6 namespace: kserve-ci-e2e-test resourceVersion: "26708" uid: 485b9284-763e-43d3-b785-51c949821d18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3bbb2f2fe045 namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 108915ff-2963-4db9-abf2-770ad9cce285 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3bbb32201a2c namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: f08d8b03-0755-465f-8327-ca51842da047 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3bbb3294088a namespace: kserve-ci-e2e-test resourceVersion: "26713" uid: 31921cd7-1806-4366-85e2-57c8bbd52d12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:49:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T17:50:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:15Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3bbb92f68d66 namespace: kserve-ci-e2e-test resourceVersion: "27248" uid: fc0dfe3a-8c0b-4cb0-b753-1befcd3b4be3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T18:06:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:06:56Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3cad17426aa5 namespace: kserve-ci-e2e-test resourceVersion: "34364" uid: 370ad894-ead7-4507-b476-05e2b303863c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: f4d44ed3-a82c-4902-8efc-b6d41630893a kind: Event lastTimestamp: "2026-07-01T18:06:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:06:56Z" name: success-200-isvc-4822a-predictor-59f89c648f-gb6hs.18be3cad1742ee39 namespace: kserve-ci-e2e-test resourceVersion: "34365" uid: 6902450f-6089-4d8e-9be4-e2201cd69db9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4822a-predictor-59f89c648f namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 33392673-f3b8-414c-a1f0-8492f58c0d72 kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: 'Created pod: success-200-isvc-4822a-predictor-59f89c648f-gb6hs' metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor-59f89c648f.18be3bbb0fe347f3 namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 45ea2cd7-5694-42e6-b5e9-a9e1397e93fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: b274c333-5a45-4d76-882f-a187b2f2fb49 kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Scaled up replica set success-200-isvc-4822a-predictor-59f89c648f from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a-predictor.18be3bbb0e09ee17 namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 16345b40-b5b7-4b3a-95f7-9e48b6b1bb16 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: fec75a27-c73d-4092-8db9-2fa05f9a8651 kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:52Z" name: success-200-isvc-4822a-predictor.18be3bbe8c8f667a namespace: kserve-ci-e2e-test resourceVersion: "27016" uid: 440790b6-94a3-430a-b5aa-e9e96f479965 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: fec75a27-c73d-4092-8db9-2fa05f9a8651 kind: Event lastTimestamp: "2026-07-01T17:49:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:49:52Z" name: success-200-isvc-4822a-predictor.18be3bbe8cfb8ab7 namespace: kserve-ci-e2e-test resourceVersion: "27018" uid: 09e988a4-3060-4053-90c7-444278d574b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:50:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27017" uid: fec75a27-c73d-4092-8db9-2fa05f9a8651 kind: Event lastTimestamp: "2026-07-01T17:50:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:52Z" name: success-200-isvc-4822a-predictor.18be3bc20bac6d5c namespace: kserve-ci-e2e-test resourceVersion: "27632" uid: 215184e2-105a-4ae3-afdd-677e26b6fb49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:50:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4822a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27017" uid: fec75a27-c73d-4092-8db9-2fa05f9a8651 kind: Event lastTimestamp: "2026-07-01T17:50:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:52Z" name: success-200-isvc-4822a-predictor.18be3bc20c123ec5 namespace: kserve-ci-e2e-test resourceVersion: "27633" uid: 3fab109d-52bc-4514-be85-8fac21c341f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4822a namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: 9917ec81-06af-47b7-af2b-466e9a6595c2 kind: Event lastTimestamp: "2026-07-01T17:50:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:25Z" name: success-200-isvc-4822a.18be3bbb0e13ce17 namespace: kserve-ci-e2e-test resourceVersion: "27333" uid: 0a3b09fa-7122-448b-91f1-760dbd1bfa44 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4822a namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: 9917ec81-06af-47b7-af2b-466e9a6595c2 kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: 'Failed to update status for InferenceService "success-200-isvc-4822a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4822a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a.18be3bbb0ea4568f namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: ded4397d-2dda-4d50-9a0f-b847b74168e9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4822a namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: 9917ec81-06af-47b7-af2b-466e9a6595c2 kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4822a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-4822a.18be3bbb0ea4a780 namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 204b8be7-aaca-498d-accf-fccd8b23ef34 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4822a namespace: kserve-ci-e2e-test resourceVersion: "27331" uid: 9917ec81-06af-47b7-af2b-466e9a6595c2 kind: Event lastTimestamp: "2026-07-01T17:50:25Z" message: InferenceService [success-200-isvc-4822a] is Ready metadata: creationTimestamp: "2026-07-01T17:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:25Z" name: success-200-isvc-4822a.18be3bc64e50bb92 namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: b252b4bb-ac46-4019-a004-1c93e8b11a70 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:29:01.194351Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6ac71-predictor-8457887695-prfhk to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3a9b4e5d13ec namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 1a28b219-27b5-4484-bc92-e9ad93d7ab97 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3a9b67f82227 namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 4c3076be-7bb8-4ccc-b7c6-029006b794af reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3a9b692c7e46 namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: 2480ab8b-9bc5-413f-870c-d75194d62910 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3a9b6bffbcbd namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 20bdc9a5-7b66-431e-aa8b-ab13a42ca1a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3a9b6c64c5ad namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 344b587c-bab0-4293-ae4d-c97e6131a9a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3a9b6c6de217 namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: 5f22acf6-7303-44e7-a0d3-f8ee82505039 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3a9b6f27c9ab namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: bc27b465-6ce4-48e1-8964-e841500a29e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3a9b6f8656c2 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: cf07cd34-dbef-48da-8943-c12f98106537 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:29:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:38:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:18Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3a9ba7fecdda namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: a63169b1-3625-4637-a14a-4819a9cf9356 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:16Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3b1c8e62eef2 namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: 983965cb-455f-4817-8eb1-a9c03363c6a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:16Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3b1c8e6406e7 namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: d1ed5e9b-0396-42c2-b391-00c8522282b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6ac71-predictor-8457887695-prfhk namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 33684255-0aa3-46d1-b986-1f2b9e4ed85c kind: Event lastTimestamp: "2026-07-01T17:38:18Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:18Z" name: success-200-isvc-6ac71-predictor-8457887695-prfhk.18be3b1d1c6f22c5 namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 5bce63fb-0105-4e47-b1b9-38663f74d814 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6ac71-predictor-8457887695 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: d0da35a2-fe14-4a54-bb84-3797707a9950 kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'Created pod: success-200-isvc-6ac71-predictor-8457887695-prfhk' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor-8457887695.18be3a9b4dfe8be9 namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: def1866f-c39d-46d1-8386-b718e1b03285 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 9fe209b5-048d-4b9b-bddc-5660f2f1a01c kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Scaled up replica set success-200-isvc-6ac71-predictor-8457887695 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71-predictor.18be3a9b4af72c6b namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: bd3ef52e-6794-4033-b0d3-ce868d21bd36 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 8e25aa53-8b5f-4309-93f9-7a3ca59d1ebb kind: Event lastTimestamp: "2026-07-01T17:29:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:31Z" name: success-200-isvc-6ac71-predictor.18be3a9ec967a20e namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: 07d3e41c-78a3-4e06-b7df-3b8e98b4f64f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 8e25aa53-8b5f-4309-93f9-7a3ca59d1ebb kind: Event lastTimestamp: "2026-07-01T17:29:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:31Z" name: success-200-isvc-6ac71-predictor.18be3a9ec9ca92ed namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: 2684aa99-0b72-4ef6-bb79-e08013c9d832 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 8e25aa53-8b5f-4309-93f9-7a3ca59d1ebb kind: Event lastTimestamp: "2026-07-01T17:30:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:01Z" name: success-200-isvc-6ac71-predictor.18be3aa5c73254f3 namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 4fe33412-2e1d-488c-86c5-9aeb7b9e22ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6ac71-predictor namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 8e25aa53-8b5f-4309-93f9-7a3ca59d1ebb kind: Event lastTimestamp: "2026-07-01T17:30:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:01Z" name: success-200-isvc-6ac71-predictor.18be3aa5c7976d14 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 6610a2db-a8a7-4e85-99f9-2eb65c61d47b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 29bf6f5c-fec3-4238-bba4-07fe6bd71cc2 kind: Event lastTimestamp: "2026-07-01T17:29:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:48Z" name: success-200-isvc-6ac71.18be3a9b4afead2f namespace: kserve-ci-e2e-test resourceVersion: "16801" uid: 346429de-04aa-4455-bc38-77238a6f5626 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 29bf6f5c-fec3-4238-bba4-07fe6bd71cc2 kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'Failed to update status for InferenceService "success-200-isvc-6ac71": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6ac71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71.18be3a9b4b87b832 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: b475d492-e9e2-4afc-beb6-da59c6121ef4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 29bf6f5c-fec3-4238-bba4-07fe6bd71cc2 kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6ac71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-6ac71.18be3a9b4b8810a9 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 5e22ecb9-e2c0-49fd-b45e-8b61b8939353 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 29bf6f5c-fec3-4238-bba4-07fe6bd71cc2 kind: Event lastTimestamp: "2026-07-01T17:29:48Z" message: InferenceService [success-200-isvc-6ac71] is Ready metadata: creationTimestamp: "2026-07-01T17:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:48Z" name: success-200-isvc-6ac71.18be3aa6632548a4 namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: e05fa713-da84-48d6-8df8-b8266e03d937 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:29:47.142243Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-766f3-predictor-788847f587-8cjdq to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3aa6011245b6 namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: 92259e2b-2f17-46b7-a227-1be2bd839e51 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3aa61acfc7c0 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: 39ac76a5-9e02-48bd-8237-30834feb07a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3aa61bffbaf8 namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 8604d558-57f7-470e-b53a-9938d2a221d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3aa61ec4fd32 namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 4df4fce8-aac7-4ec6-ab4d-38d57521c66b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3aa61f24ff09 namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 2d1d3e73-d918-4f9b-a538-c63858dc5945 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3aa61f2ea096 namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: 34a6f2da-145c-4cd9-b4cd-57bd325dcaa7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3aa621ea5983 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: e750b2c6-d18a-45b9-9b1e-e9ec8627ac1b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3aa6227922d3 namespace: kserve-ci-e2e-test resourceVersion: "16775" uid: 6aeb7f9c-d708-4e07-956d-db93e35c7cf4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:39:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:03Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3aa62ec4a886 namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: 6a6ac9f5-3518-4f7f-a7a8-870e5006e48b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:39:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:01Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3b272c57d1fb namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: c3072e7e-ee10-43e3-af52-cabbb0fcde16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:39:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:01Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3b272c586a92 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 7e1c16f1-0fe9-4089-9914-ff639dcc1604 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-766f3-predictor-788847f587-8cjdq namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: aaf6e27c-982c-4694-a89c-b49ac00d38e1 kind: Event lastTimestamp: "2026-07-01T17:39:03Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:03Z" name: success-200-isvc-766f3-predictor-788847f587-8cjdq.18be3b27a32c958a namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: 583386ac-b24c-4679-a678-9d1eda80dadb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-766f3-predictor-788847f587 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: 3712c5bf-1a65-4320-bc1e-30ee46bd530f kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'Created pod: success-200-isvc-766f3-predictor-788847f587-8cjdq' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor-788847f587.18be3aa6005d3374 namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: 5540022c-a425-47fc-9277-d04c97e4c0ef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: bff98a4e-2f24-43f1-8345-7d3e7f25cd9f kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: Scaled up replica set success-200-isvc-766f3-predictor-788847f587 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3-predictor.18be3aa5fe7a6898 namespace: kserve-ci-e2e-test resourceVersion: "16690" uid: e77acd1b-acb7-4256-bf8f-79277ee310e4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 7ce04108-9446-4a60-a228-7d77090d7ff0 kind: Event lastTimestamp: "2026-07-01T17:30:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:17Z" name: success-200-isvc-766f3-predictor.18be3aa97d1fc8a1 namespace: kserve-ci-e2e-test resourceVersion: "17092" uid: d606a577-e93c-4799-9d4d-4faed6f344f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:30:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 7ce04108-9446-4a60-a228-7d77090d7ff0 kind: Event lastTimestamp: "2026-07-01T17:30:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:17Z" name: success-200-isvc-766f3-predictor.18be3aa97d8499d3 namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: 3485425d-df64-45df-a27f-88e29a1912e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: 7ce04108-9446-4a60-a228-7d77090d7ff0 kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:02Z" name: success-200-isvc-766f3-predictor.18be3ab07af73c9c namespace: kserve-ci-e2e-test resourceVersion: "17475" uid: 839c8e8a-961d-4756-8b5f-b9374378ed7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-766f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: 7ce04108-9446-4a60-a228-7d77090d7ff0 kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:02Z" name: success-200-isvc-766f3-predictor.18be3ab07b5ba473 namespace: kserve-ci-e2e-test resourceVersion: "17476" uid: 9eaa7814-3a5e-463b-a160-d2c60c91a1a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-766f3 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 9e2ef567-197c-4da7-8235-3bdf0b25da98 kind: Event lastTimestamp: "2026-07-01T17:30:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:34Z" name: success-200-isvc-766f3.18be3aa5feb1b39d namespace: kserve-ci-e2e-test resourceVersion: "17228" uid: 0beb33d4-68f9-454d-b213-ecf3d940c0d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-766f3 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 9e2ef567-197c-4da7-8235-3bdf0b25da98 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-766f3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-766f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3.18be3aa5ff397c9e namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: 9f03cea1-eed1-4a64-937a-40776692d033 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-766f3 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 9e2ef567-197c-4da7-8235-3bdf0b25da98 kind: Event lastTimestamp: "2026-07-01T17:29:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-766f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:47Z" name: success-200-isvc-766f3.18be3aa5ff39c6d3 namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 52461208-9100-4474-9606-4446e9deb180 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-766f3 namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: 9e2ef567-197c-4da7-8235-3bdf0b25da98 kind: Event lastTimestamp: "2026-07-01T17:30:34Z" message: InferenceService [success-200-isvc-766f3] is Ready metadata: creationTimestamp: "2026-07-01T17:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:34Z" name: success-200-isvc-766f3.18be3ab0ea1d4a33 namespace: kserve-ci-e2e-test resourceVersion: "17226" uid: 896e3510-d0ed-43db-9f75-19e05790ac97 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:40:12.189295Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:40:12Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b3788c76b20 namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: 11d5c953-92b3-491d-8f5b-71659e5aeb6c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8b57f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:12Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b3791fe0709 namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: 33865fe1-92d6-42b8-8115-29192e6dbc5f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:40:13Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b37c75012a8 namespace: kserve-ci-e2e-test resourceVersion: "22043" uid: dc91e435-7b80-40ab-9f39-b7c7a87a897f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b37c8b43ca7 namespace: kserve-ci-e2e-test resourceVersion: "22045" uid: 6f34b5aa-7175-4a5e-8b3d-5ae87b42df5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b37cc354580 namespace: kserve-ci-e2e-test resourceVersion: "22046" uid: b5ca1b0c-56e4-4027-851a-3ff15120ab99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b37ccb8edb5 namespace: kserve-ci-e2e-test resourceVersion: "22048" uid: 1f0f0754-6031-4d01-91b4-37bd06ad0483 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b37ccc5af4b namespace: kserve-ci-e2e-test resourceVersion: "22049" uid: 439b260e-802f-4845-8227-d790dd017fe6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b37d09100ba namespace: kserve-ci-e2e-test resourceVersion: "22052" uid: 1cd9ccb3-253f-40f4-a04e-58c1a4286e75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:40:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:13Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b37d10c4415 namespace: kserve-ci-e2e-test resourceVersion: "22054" uid: cc2f07b8-32d8-4c79-b7fa-7ed495d5f6c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T17:40:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:49:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:39Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3b37e4dc1da9 namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: e79a52e4-0bd3-4934-9475-433589002d8a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3bbb0d5c53dc namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: 954e381a-2f8b-41c6-86bf-9fe02f102bb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:49:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:37Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3bbb0d5cd756 namespace: kserve-ci-e2e-test resourceVersion: "26633" uid: 63b9cfdd-2321-4aa8-8fb0-6303ce3a94e4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:49:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: bb5148e0-2644-410d-b878-dc7183e9e3bb kind: Event lastTimestamp: "2026-07-01T17:49:39Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:49:39Z" name: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq.18be3bbbad7e127e namespace: kserve-ci-e2e-test resourceVersion: "26745" uid: 0d7d19ce-fb28-48a1-a36a-d1860f93752c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8b57f-predictor-68d7874c4c namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: ba55dca4-0ab5-4eb8-9cf7-4575a3ae08c7 kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: 'Created pod: success-200-isvc-8b57f-predictor-68d7874c4c-h6pdq' metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:12Z" name: success-200-isvc-8b57f-predictor-68d7874c4c.18be3b37885a5047 namespace: kserve-ci-e2e-test resourceVersion: "21973" uid: 32c0da50-a4b7-4e82-993c-a08d7e540474 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: cfb72e67-1df8-423c-bedf-69d183ef32da kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: Scaled up replica set success-200-isvc-8b57f-predictor-68d7874c4c from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:12Z" name: success-200-isvc-8b57f-predictor.18be3b3786846d78 namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: 9101f23c-63bb-4ad3-bcfd-c228f60bff57 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:40:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: cae0cb24-ac45-4d78-85e5-1214982a7017 kind: Event lastTimestamp: "2026-07-01T17:40:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:57Z" name: success-200-isvc-8b57f-predictor.18be3b3b04c3e9f9 namespace: kserve-ci-e2e-test resourceVersion: "22480" uid: fc50d35e-9297-401a-8c57-0938100521c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:40:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: cae0cb24-ac45-4d78-85e5-1214982a7017 kind: Event lastTimestamp: "2026-07-01T17:40:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:57Z" name: success-200-isvc-8b57f-predictor.18be3b3b05298e1b namespace: kserve-ci-e2e-test resourceVersion: "22481" uid: dd2e20ba-4caf-4bbc-b22c-32c40500e9ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "22225" uid: cae0cb24-ac45-4d78-85e5-1214982a7017 kind: Event lastTimestamp: "2026-07-01T17:41:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:42Z" name: success-200-isvc-8b57f-predictor.18be3b3e83d80af7 namespace: kserve-ci-e2e-test resourceVersion: "22858" uid: 33e46c7d-7caf-4d74-b6e9-c0eb4ac941ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8b57f-predictor namespace: kserve-ci-e2e-test resourceVersion: "22225" uid: cae0cb24-ac45-4d78-85e5-1214982a7017 kind: Event lastTimestamp: "2026-07-01T17:41:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:41:42Z" name: success-200-isvc-8b57f-predictor.18be3b3e843df611 namespace: kserve-ci-e2e-test resourceVersion: "22859" uid: bbbb1c00-3ccc-458e-bd93-5542684a5f88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b57f namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: 0186b482-e370-4565-a847-8d62ab0b9439 kind: Event lastTimestamp: "2026-07-01T17:41:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:41:09Z" name: success-200-isvc-8b57f.18be3b37868d1f7d namespace: kserve-ci-e2e-test resourceVersion: "22590" uid: af7d7aac-8bab-41b1-bac2-29fe4700f524 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b57f namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: 0186b482-e370-4565-a847-8d62ab0b9439 kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-8b57f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8b57f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:12Z" name: success-200-isvc-8b57f.18be3b37872cb3d3 namespace: kserve-ci-e2e-test resourceVersion: "21966" uid: 0226d890-2e90-4eda-afde-575fc0261b14 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b57f namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: 0186b482-e370-4565-a847-8d62ab0b9439 kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8b57f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:12Z" name: success-200-isvc-8b57f.18be3b37872d212b namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: a3e2f213-502f-483f-be86-919d7489c2ed reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:41:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8b57f namespace: kserve-ci-e2e-test resourceVersion: "22588" uid: 0186b482-e370-4565-a847-8d62ab0b9439 kind: Event lastTimestamp: "2026-07-01T17:41:09Z" message: InferenceService [success-200-isvc-8b57f] is Ready metadata: creationTimestamp: "2026-07-01T17:41:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:41:09Z" name: success-200-isvc-8b57f.18be3b44f476704b namespace: kserve-ci-e2e-test resourceVersion: "22589" uid: f633d88f-29cc-4468-ba7e-65dda8b1db24 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:39:01.957298Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:39:01Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b272ea02b7d namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 51ac99bc-e4c8-4516-b12d-2fcf35f5df84 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:39:02Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b274838c39c namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: 21e9ae2e-6162-4b1c-aa7e-673fa625e1f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:02Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b2749788952 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: cadf8ddd-515d-4c0c-b644-1e6890e6bce3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:02Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b274c507e83 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8569cb41-b567-4c52-b24b-18a75a62a56f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:02Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b274cbb1678 namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: 3f9e39cc-cd44-4f71-b431-1d99ba864ebc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:02Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b274cc3ce44 namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: 80d0004f-7ec8-4fa9-acf9-bd70b22c9e20 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:02Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b274f956d5d namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: 5533f120-db74-4545-9327-5fb08330755a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:39:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:02Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b27502104d9 namespace: kserve-ci-e2e-test resourceVersion: "21070" uid: fccc2102-f095-47ab-9a3b-df857a5283fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:39:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:40Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b278475e1ca namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 60b464c6-9505-4d34-a65c-d155381f3693 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:12Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b3787b0d0be namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: cc9260bd-3b93-4498-8580-13242b5cf1d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:40:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:12Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b3787b19a4b namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: 18bdd084-fe7d-41c8-ba45-d249dc894d6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 42c4052a-d324-468b-9f06-7585e46b5a0e kind: Event lastTimestamp: "2026-07-01T17:40:15Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:15Z" name: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs.18be3b384892e724 namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: 804467a4-b478-48df-8ac9-37169f1d0d91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-98f1c-predictor-659b84d7f7 namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 458f8334-70c2-48aa-87f2-e589abd04015 kind: Event lastTimestamp: "2026-07-01T17:39:01Z" message: 'Created pod: success-200-isvc-98f1c-predictor-659b84d7f7-f2vvs' metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:01Z" name: success-200-isvc-98f1c-predictor-659b84d7f7.18be3b272e2e9f9e namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: e61af0dd-9908-47d1-9b08-188826e6ce6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-98f1c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 61a90ae7-abb4-4a88-86dd-03c1d7fc923f kind: Event lastTimestamp: "2026-07-01T17:39:01Z" message: Scaled up replica set success-200-isvc-98f1c-predictor-659b84d7f7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:01Z" name: success-200-isvc-98f1c-predictor.18be3b272c6b2ed9 namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 42a74410-6744-424d-b64e-b7f69f2b08ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:39:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-98f1c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20984" uid: 4681c937-6f45-40a1-aab5-23970aa5ba4b kind: Event lastTimestamp: "2026-07-01T17:40:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:01Z" name: success-200-isvc-98f1c-predictor.18be3b2aab4edaff namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 47ba4ef1-8a62-45ae-8b8f-cb80601b7215 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:39:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-98f1c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20984" uid: 4681c937-6f45-40a1-aab5-23970aa5ba4b kind: Event lastTimestamp: "2026-07-01T17:40:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:01Z" name: success-200-isvc-98f1c-predictor.18be3b2aabb4e96d namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: f82be55a-686f-4720-84a1-3b86c748dd88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-98f1c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: 4681c937-6f45-40a1-aab5-23970aa5ba4b kind: Event lastTimestamp: "2026-07-01T17:39:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:31Z" name: success-200-isvc-98f1c-predictor.18be3b2e2a5d51e2 namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: f8df6f53-09d3-42d8-8713-f3b3784252ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-98f1c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: 4681c937-6f45-40a1-aab5-23970aa5ba4b kind: Event lastTimestamp: "2026-07-01T17:39:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:31Z" name: success-200-isvc-98f1c-predictor.18be3b2e2ac306a8 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 7295cbab-fa77-49c0-b3c8-01e83eee122c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-98f1c namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 5a52f736-80b0-4ffa-9098-02380eb024ad kind: Event lastTimestamp: "2026-07-01T17:39:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:50Z" name: success-200-isvc-98f1c.18be3b272cd2a8d8 namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 82d9d554-e44b-4621-abcb-af8514a6db89 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-98f1c namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 5a52f736-80b0-4ffa-9098-02380eb024ad kind: Event lastTimestamp: "2026-07-01T17:39:01Z" message: 'Failed to update status for InferenceService "success-200-isvc-98f1c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-98f1c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:01Z" name: success-200-isvc-98f1c.18be3b272d3ef405 namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: ff7d578e-11b8-48c8-bf55-97dae1516687 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-98f1c namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 5a52f736-80b0-4ffa-9098-02380eb024ad kind: Event lastTimestamp: "2026-07-01T17:39:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-98f1c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:01Z" name: success-200-isvc-98f1c.18be3b272d3f2916 namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 6391b9fc-b822-4bfc-aced-fdfe869f2996 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-98f1c namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: 5a52f736-80b0-4ffa-9098-02380eb024ad kind: Event lastTimestamp: "2026-07-01T17:39:50Z" message: InferenceService [success-200-isvc-98f1c] is Ready metadata: creationTimestamp: "2026-07-01T17:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:50Z" name: success-200-isvc-98f1c.18be3b327c048c5f namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: ebb47389-7f24-4b2d-8935-23dc77c991df reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:39:26.451681Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c4578-predictor-59bdd4495c-5lkdr to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3b2ce29abba1 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 20478247-6f28-469d-beda-dfa45caf275f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3b2cfc70da4a namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 778930e4-c0a5-4e26-8524-b22b518f71fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" already present on machine metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3b2cfdee440d namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: 936ec9d3-9e7e-43f2-b3f0-bad28b54c4e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3b2d00f9b949 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: 1f3f2661-3e2c-4b6b-9e25-857046244af1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3b2d01a3ee85 namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: 900dcbbd-04e0-484d-a64b-709431209d10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3b2d01ae18b9 namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: b5d862b3-ce69-49e9-99df-a67fc5fed2a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:39:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:27Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3b2d04a8baa0 namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: c6eb5683-6dc8-44d3-b979-2efadbf9a154 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:39:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:39:27Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3b2d0519cb9f namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: b3953f62-80e8-43c0-8f45-e80ed7c62ddc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:39:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:48:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:44Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3b2d5e05e1e3 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 50d6c2cd-7b6a-40ff-948a-29fa9c63f351 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3bae1a365ef6 namespace: kserve-ci-e2e-test resourceVersion: "26060" uid: 65f878dd-b213-432c-b50b-acca4c99682d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:48:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:41Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3bae1a390748 namespace: kserve-ci-e2e-test resourceVersion: "26064" uid: 8fa394e8-21e9-4e8c-bb6a-15e6923f2487 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 23993103-8201-45fe-b882-e46ccab1da61 kind: Event lastTimestamp: "2026-07-01T17:48:44Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:48:44Z" name: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr.18be3baed26f282a namespace: kserve-ci-e2e-test resourceVersion: "26172" uid: 11df2fbd-087e-464d-88c5-2b75053ac810 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c4578-predictor-59bdd4495c namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 9279c48e-ea20-484d-8624-9544fc0df3e8 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: 'Created pod: success-200-isvc-c4578-predictor-59bdd4495c-5lkdr' metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578-predictor-59bdd4495c.18be3b2ce1f403af namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: a81ef6b6-373d-46d2-9076-3c40460053d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 524ea873-6ec0-45a9-9b09-d2748884a902 kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: Scaled up replica set success-200-isvc-c4578-predictor-59bdd4495c from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578-predictor.18be3b2ce002822f namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 34ecadbb-530c-4a93-bfb5-7aa60513a0c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:39:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: b8558a0a-23a5-4f3d-b53d-c1527ebe9d21 kind: Event lastTimestamp: "2026-07-01T17:39:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:56Z" name: success-200-isvc-c4578-predictor.18be3b305ec9e408 namespace: kserve-ci-e2e-test resourceVersion: "21748" uid: 63377c48-b693-43da-89fb-31fa35adee6f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:39:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: b8558a0a-23a5-4f3d-b53d-c1527ebe9d21 kind: Event lastTimestamp: "2026-07-01T17:39:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:39:56Z" name: success-200-isvc-c4578-predictor.18be3b305f323845 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 02964b8a-ad7a-4792-bb4d-b0a3ea7dbf10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: b8558a0a-23a5-4f3d-b53d-c1527ebe9d21 kind: Event lastTimestamp: "2026-07-01T17:40:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:41Z" name: success-200-isvc-c4578-predictor.18be3b375d2a1a2a namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: f18bf19e-019a-439f-85bd-3b85be588d03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c4578-predictor namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: b8558a0a-23a5-4f3d-b53d-c1527ebe9d21 kind: Event lastTimestamp: "2026-07-01T17:40:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:40:41Z" name: success-200-isvc-c4578-predictor.18be3b375d904a08 namespace: kserve-ci-e2e-test resourceVersion: "22352" uid: 15fa44e8-ef86-4f01-86ae-dc162aeaac54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c4578 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 52236baa-cdb9-4144-9aee-c57cb82d0cda kind: Event lastTimestamp: "2026-07-01T17:40:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:14Z" name: success-200-isvc-c4578.18be3b2ce0177c63 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: 42031134-7fbf-4fa8-bd35-d709921db612 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c4578 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 52236baa-cdb9-4144-9aee-c57cb82d0cda kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-c4578": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c4578": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578.18be3b2ce0d6bdea namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: 4cb337bd-5a34-4519-969a-87aefa1cdee3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:39:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c4578 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 52236baa-cdb9-4144-9aee-c57cb82d0cda kind: Event lastTimestamp: "2026-07-01T17:39:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c4578": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:39:26Z" name: success-200-isvc-c4578.18be3b2ce0d70ff9 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 98ba473a-0f73-4e8f-a3d0-c15d0ee1951a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c4578 namespace: kserve-ci-e2e-test resourceVersion: "22087" uid: 52236baa-cdb9-4144-9aee-c57cb82d0cda kind: Event lastTimestamp: "2026-07-01T17:40:14Z" message: InferenceService [success-200-isvc-c4578] is Ready metadata: creationTimestamp: "2026-07-01T17:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:40:14Z" name: success-200-isvc-c4578.18be3b3819a5c6d6 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: 7372cde2-ba0e-41e0-a15e-ce28e36de81c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:27:27.067921Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz to ip-10-0-139-93.ec2.internal metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:27:27Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a8563fdbf8a namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: 39dcbf2e-59f7-41ad-a9ee-44ed0191be37 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:27:27Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a857e772151 namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: c5468840-e4de-4a84-97a7-6b36061dc141 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:27Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a857fd0bf11 namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: b5916a9a-e4c0-416e-b1b0-6777994b9ee3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:27:40Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1692" in 13.07s (13.07s including waiting). Image size: 1337514096 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:40Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a888ade6328 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: ecb585b8-8955-44cb-8703-6e1c00d1eb3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:27:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:40Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a888df1e22a namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: d79cad2f-d801-4f3f-92f1-28a4e0fd1f28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:27:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:40Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a888e5d6dbc namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: fc21d7d6-3f68-4bb6-acbc-8240410083ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:27:40Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:40Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a888e68ae6a namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: b8e75c22-66ac-4cda-b1d9-7ebf74430a2b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:27:42Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.222s (2.222s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:42Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a8912dfdb45 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 952c1c24-08fd-44eb-bdd3-7be4d3ca1b49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:27:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:42Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a89171732d4 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 377dee30-cd6d-4b23-b8ec-b0059108dafb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:27:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:27:42Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a8917a06bf0 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 77022860-5564-4193-9485-417777f2627c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:27:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:28:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:28:19Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a892d6b910e namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: e93934b7-0f47-45c6-b4d0-a636b257fd42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a9b4fff85a0 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: ca544d4f-40d8-4538-bbb8-6974959bc9f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: cdcce57f-3beb-410d-84df-57c7c0b30905 kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz.18be3a9b5000a03c namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: f843b9bd-4936-4478-b06f-daf4e2366457 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-93.ec2.internal source: component: kubelet host: ip-10-0-139-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d0081-predictor-6b7b4d678f namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 8d84bd63-3af8-4d99-bbe4-f072f150b89d kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: 'Created pod: success-200-isvc-d0081-predictor-6b7b4d678f-x5ffz' metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:27Z" name: success-200-isvc-d0081-predictor-6b7b4d678f.18be3a85638d3001 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: d3eaf9d5-714b-4cbb-ada6-ef4c802c43dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: 8f2365ca-fc76-492b-844b-cf03d4426c8d kind: Event lastTimestamp: "2026-07-01T17:27:27Z" message: Scaled up replica set success-200-isvc-d0081-predictor-6b7b4d678f from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:27:27Z" name: success-200-isvc-d0081-predictor.18be3a8560f30bc5 namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: e96ad962-9e5f-4519-a9c5-b02fc5afa45e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 3f0f6885-91ad-453a-8b19-2cc39c24b33b kind: Event lastTimestamp: "2026-07-01T17:28:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:27Z" name: success-200-isvc-d0081-predictor.18be3a88df8b5a97 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: b7f917a8-33b2-4598-a30a-d8b4fe8559c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 3f0f6885-91ad-453a-8b19-2cc39c24b33b kind: Event lastTimestamp: "2026-07-01T17:28:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:27Z" name: success-200-isvc-d0081-predictor.18be3a88dfff80f7 namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: fa10baed-a0ff-4d30-a276-aee2e5d3a4cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:28:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 3f0f6885-91ad-453a-8b19-2cc39c24b33b kind: Event lastTimestamp: "2026-07-01T17:28:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:57Z" name: success-200-isvc-d0081-predictor.18be3a96dacb9308 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 40fbe2e1-50f0-4a0d-b39f-f068a71b7c64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:28:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0081-predictor namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 3f0f6885-91ad-453a-8b19-2cc39c24b33b kind: Event lastTimestamp: "2026-07-01T17:28:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:57Z" name: success-200-isvc-d0081-predictor.18be3a96db338e71 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: 6890c29d-ce9d-4f71-9c13-994a7b858c94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0081 namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 3c47af85-4fb3-4c37-b418-852fcf4ec552 kind: Event lastTimestamp: "2026-07-01T17:28:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:28:29Z" name: success-200-isvc-d0081.18be3a85610a54fc namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: eb65bec4-e660-4840-a64f-08352a706f2f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0081 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 3c47af85-4fb3-4c37-b418-852fcf4ec552 kind: Event lastTimestamp: "2026-07-01T17:28:29Z" message: InferenceService [success-200-isvc-d0081] is Ready metadata: creationTimestamp: "2026-07-01T17:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:28:29Z" name: success-200-isvc-d0081.18be3a93e92975ec namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: ef42685f-1e6c-4ba2-a32e-9dca7b72096e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:50:37.240500Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4822a-76849cbcd4-cg6kj namespace: kserve-ci-e2e-test resourceVersion: "27453" uid: 70eef7eb-09d3-4f6d-98a1-3eeb00e09d7f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-4822a-76849cbcd4-cg6kj to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:50:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:50:37Z" name: switch-graph-4822a-76849cbcd4-cg6kj.18be3bc910bbe01b namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 573570df-3519-4355-84be-870c17bba86b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4822a-76849cbcd4-cg6kj namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: 70eef7eb-09d3-4f6d-98a1-3eeb00e09d7f kind: Event lastTimestamp: "2026-07-01T17:50:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-4822a-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:37Z" name: switch-graph-4822a-76849cbcd4-cg6kj.18be3bc919e45e5a namespace: kserve-ci-e2e-test resourceVersion: "27473" uid: 97b819b0-16e0-4b37-a4c9-204593fe9e95 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4822a-76849cbcd4-cg6kj namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: 70eef7eb-09d3-4f6d-98a1-3eeb00e09d7f kind: Event lastTimestamp: "2026-07-01T17:50:38Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:50:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:50:38Z" name: switch-graph-4822a-76849cbcd4-cg6kj.18be3bc94e347c8c namespace: kserve-ci-e2e-test resourceVersion: "27481" uid: 62992ac1-59cd-4216-ad9d-bdc331d256f5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4822a} kind: Pod name: switch-graph-4822a-76849cbcd4-cg6kj namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: 70eef7eb-09d3-4f6d-98a1-3eeb00e09d7f kind: Event lastTimestamp: "2026-07-01T17:50:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:50:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:38Z" name: switch-graph-4822a-76849cbcd4-cg6kj.18be3bc94fa13401 namespace: kserve-ci-e2e-test resourceVersion: "27483" uid: e5cc232d-2b29-47b1-a546-90a3e6a8cf04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4822a} kind: Pod name: switch-graph-4822a-76849cbcd4-cg6kj namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: 70eef7eb-09d3-4f6d-98a1-3eeb00e09d7f kind: Event lastTimestamp: "2026-07-01T17:50:38Z" message: 'Created container: switch-graph-4822a' metadata: creationTimestamp: "2026-07-01T17:50:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:38Z" name: switch-graph-4822a-76849cbcd4-cg6kj.18be3bc952ba250b namespace: kserve-ci-e2e-test resourceVersion: "27485" uid: c8aae544-5123-43f4-a96f-ecf38b4b00cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4822a} kind: Pod name: switch-graph-4822a-76849cbcd4-cg6kj namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: 70eef7eb-09d3-4f6d-98a1-3eeb00e09d7f kind: Event lastTimestamp: "2026-07-01T17:50:38Z" message: Started container switch-graph-4822a metadata: creationTimestamp: "2026-07-01T17:50:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:50:38Z" name: switch-graph-4822a-76849cbcd4-cg6kj.18be3bc9535295dc namespace: kserve-ci-e2e-test resourceVersion: "27486" uid: 32cdbc54-345b-4500-a699-62ca641530d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4822a} kind: Pod name: switch-graph-4822a-76849cbcd4-cg6kj namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: 70eef7eb-09d3-4f6d-98a1-3eeb00e09d7f kind: Event lastTimestamp: "2026-07-01T18:06:56Z" message: Stopping container switch-graph-4822a metadata: creationTimestamp: "2026-07-01T18:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:06:56Z" name: switch-graph-4822a-76849cbcd4-cg6kj.18be3cad0f2d1525 namespace: kserve-ci-e2e-test resourceVersion: "34348" uid: f8ae054a-2f8d-4b16-98c9-55161a5f5919 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:07:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4822a} kind: Pod name: switch-graph-4822a-76849cbcd4-cg6kj namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: 70eef7eb-09d3-4f6d-98a1-3eeb00e09d7f kind: Event lastTimestamp: "2026-07-01T18:07:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:07:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:07:20Z" name: switch-graph-4822a-76849cbcd4-cg6kj.18be3cadf1c990b7 namespace: kserve-ci-e2e-test resourceVersion: "34587" uid: b2f78d94-09b0-449a-ab76-132107ba22ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-4822a-76849cbcd4 namespace: kserve-ci-e2e-test resourceVersion: "27449" uid: dd0e033c-c217-4707-8b42-90b5f4ece626 kind: Event lastTimestamp: "2026-07-01T17:50:37Z" message: 'Created pod: switch-graph-4822a-76849cbcd4-cg6kj' metadata: creationTimestamp: "2026-07-01T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:37Z" name: switch-graph-4822a-76849cbcd4.18be3bc9100e7ef5 namespace: kserve-ci-e2e-test resourceVersion: "27455" uid: f63f6391-4d4e-479b-8723-9307ad844553 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-4822a namespace: kserve-ci-e2e-test resourceVersion: "27447" uid: dce27d96-32f4-41ff-b440-cbec98287dcf kind: Event lastTimestamp: "2026-07-01T17:50:37Z" message: Scaled up replica set switch-graph-4822a-76849cbcd4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:50:37Z" name: switch-graph-4822a.18be3bc90d342edc namespace: kserve-ci-e2e-test resourceVersion: "27450" uid: a3455f21-1dcc-438b-bf0f-815bf8ca094e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4822a namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: cb7c98fd-6743-48a4-939b-8afb726bade9 kind: Event lastTimestamp: "2026-07-01T17:50:37Z" message: InferenceGraph [switch-graph-4822a] is Ready metadata: creationTimestamp: "2026-07-01T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:37Z" name: switch-graph-4822a.18be3bc914a37ef9 namespace: kserve-ci-e2e-test resourceVersion: "27468" uid: ccd8fa7f-50f7-4f1b-93de-cd0ec17b15f1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4822a namespace: kserve-ci-e2e-test resourceVersion: "27439" uid: cb7c98fd-6743-48a4-939b-8afb726bade9 kind: Event lastTimestamp: "2026-07-01T17:50:37Z" message: 'Failed to update status for InferenceGraph "switch-graph-4822a": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-4822a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:37Z" name: switch-graph-4822a.18be3bc916f6bd13 namespace: kserve-ci-e2e-test resourceVersion: "27469" uid: 2db7555e-235c-46f1-aa41-877d71b19a20 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:50:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4822a namespace: kserve-ci-e2e-test resourceVersion: "27439" uid: cb7c98fd-6743-48a4-939b-8afb726bade9 kind: Event lastTimestamp: "2026-07-01T17:50:37Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-4822a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:50:37Z" name: switch-graph-4822a.18be3bc916f70eee namespace: kserve-ci-e2e-test resourceVersion: "27470" uid: ac8a876f-2262-4cd9-98c5-db2a141346ea reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:50:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4822a namespace: kserve-ci-e2e-test resourceVersion: "27448" uid: a2583e62-9276-4d41-87c1-187927cd69db kind: Event lastTimestamp: "2026-07-01T17:51:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:07Z" name: switch-graph-4822a.18be3bcc8b4ed361 namespace: kserve-ci-e2e-test resourceVersion: "27763" uid: ea962ac7-6b00-49d6-a540-8ce19e2f93f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:50:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4822a namespace: kserve-ci-e2e-test resourceVersion: "27448" uid: a2583e62-9276-4d41-87c1-187927cd69db kind: Event lastTimestamp: "2026-07-01T17:51:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:50:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:51:07Z" name: switch-graph-4822a.18be3bcc8b94b183 namespace: kserve-ci-e2e-test resourceVersion: "27764" uid: d3dbcc1b-5eab-4f46-b8d6-d4958f3e85d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:30:01.239835Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6ac71-9d4758597-gcf7j namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 40577f87-46c7-483d-a615-428687ecf35b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-6ac71-9d4758597-gcf7j to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:30:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:30:01Z" name: switch-graph-6ac71-9d4758597-gcf7j.18be3aa9495a6a5c namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: 2d7b37be-ee2e-40a2-be00-9257c6e5a88a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6ac71-9d4758597-gcf7j namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 40577f87-46c7-483d-a615-428687ecf35b kind: Event lastTimestamp: "2026-07-01T17:30:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-6ac71-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:01Z" name: switch-graph-6ac71-9d4758597-gcf7j.18be3aa95773e9f8 namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: 380f8097-03e1-4663-ae58-3145b74b4968 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:02Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6ac71-9d4758597-gcf7j namespace: kserve-ci-e2e-test resourceVersion: "16934" uid: 40577f87-46c7-483d-a615-428687ecf35b kind: Event lastTimestamp: "2026-07-01T17:30:02Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:30:02Z" name: switch-graph-6ac71-9d4758597-gcf7j.18be3aa987fea1ff namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: 7aafa3ef-e468-47bc-90c4-5a24be6bafd7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6ac71} kind: Pod name: switch-graph-6ac71-9d4758597-gcf7j namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 40577f87-46c7-483d-a615-428687ecf35b kind: Event lastTimestamp: "2026-07-01T17:30:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" already present on machine metadata: creationTimestamp: "2026-07-01T17:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:02Z" name: switch-graph-6ac71-9d4758597-gcf7j.18be3aa9894cbd19 namespace: kserve-ci-e2e-test resourceVersion: "16961" uid: 6b0e1fdf-1825-4c92-b841-1a9d2e7074bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6ac71} kind: Pod name: switch-graph-6ac71-9d4758597-gcf7j namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 40577f87-46c7-483d-a615-428687ecf35b kind: Event lastTimestamp: "2026-07-01T17:30:02Z" message: 'Created container: switch-graph-6ac71' metadata: creationTimestamp: "2026-07-01T17:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:02Z" name: switch-graph-6ac71-9d4758597-gcf7j.18be3aa98c310611 namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: 4181c1bd-23f4-45a6-8a59-0c66d61c1651 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6ac71} kind: Pod name: switch-graph-6ac71-9d4758597-gcf7j namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 40577f87-46c7-483d-a615-428687ecf35b kind: Event lastTimestamp: "2026-07-01T17:30:02Z" message: Started container switch-graph-6ac71 metadata: creationTimestamp: "2026-07-01T17:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:02Z" name: switch-graph-6ac71-9d4758597-gcf7j.18be3aa98c963389 namespace: kserve-ci-e2e-test resourceVersion: "16963" uid: 68c216fb-7623-48a3-8028-a2300d27cf3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:38:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6ac71} kind: Pod name: switch-graph-6ac71-9d4758597-gcf7j namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 40577f87-46c7-483d-a615-428687ecf35b kind: Event lastTimestamp: "2026-07-01T17:38:16Z" message: Stopping container switch-graph-6ac71 metadata: creationTimestamp: "2026-07-01T17:38:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:16Z" name: switch-graph-6ac71-9d4758597-gcf7j.18be3b1c7ce0eab0 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 55278690-436f-420d-be7b-15a44ad9eb41 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6ac71} kind: Pod name: switch-graph-6ac71-9d4758597-gcf7j namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 40577f87-46c7-483d-a615-428687ecf35b kind: Event lastTimestamp: "2026-07-01T17:38:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:38:44Z" name: switch-graph-6ac71-9d4758597-gcf7j.18be3b1d3f6be3a0 namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 8cd74b03-fc77-4235-a4e1-bd4d8a697153 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-6ac71-9d4758597 namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: 6aa15815-f4fc-4ac7-9481-70f55033641f kind: Event lastTimestamp: "2026-07-01T17:30:01Z" message: 'Created pod: switch-graph-6ac71-9d4758597-gcf7j' metadata: creationTimestamp: "2026-07-01T17:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:01Z" name: switch-graph-6ac71-9d4758597.18be3aa948e56f02 namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: bcad5977-6be7-435f-bbb4-0339495d72db reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16918" uid: 363afa6e-6da1-4fa9-8831-2f4c6b550eae kind: Event lastTimestamp: "2026-07-01T17:30:01Z" message: Scaled up replica set switch-graph-6ac71-9d4758597 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:01Z" name: switch-graph-6ac71.18be3aa948319d38 namespace: kserve-ci-e2e-test resourceVersion: "16922" uid: 97a16db0-7f82-4148-b93c-7bc224e0fc8f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: e1ab1dca-d4a1-443c-a21c-93c418de9e79 kind: Event lastTimestamp: "2026-07-01T17:30:01Z" message: InferenceGraph [switch-graph-6ac71] is Ready metadata: creationTimestamp: "2026-07-01T17:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:01Z" name: switch-graph-6ac71.18be3aa94da34163 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: 3c94146b-490d-41c9-99e9-7ea6adfe5c5c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 05678444-7926-4e81-aa74-098befe8b82c kind: Event lastTimestamp: "2026-07-01T17:30:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:16Z" name: switch-graph-6ac71.18be3aacc6993978 namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: 4b66c86c-271c-40a7-9ac6-a56a8bcdaa3c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 05678444-7926-4e81-aa74-098befe8b82c kind: Event lastTimestamp: "2026-07-01T17:30:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:16Z" name: switch-graph-6ac71.18be3aacc6fde08d namespace: kserve-ci-e2e-test resourceVersion: "17082" uid: 3ca94426-075e-4687-90eb-e19f40e8efe4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "17081" uid: 05678444-7926-4e81-aa74-098befe8b82c kind: Event lastTimestamp: "2026-07-01T17:30:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:31Z" name: switch-graph-6ac71.18be3ab0459a893c namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 6a430a68-8370-411f-86af-5e3d6e76f534 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6ac71 namespace: kserve-ci-e2e-test resourceVersion: "17081" uid: 05678444-7926-4e81-aa74-098befe8b82c kind: Event lastTimestamp: "2026-07-01T17:30:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:31Z" name: switch-graph-6ac71.18be3ab045fa57f9 namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: 8e4b0eed-34dc-4802-bee7-dd2578722c5f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:28:46.898858Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d0081-685667769f-jhtmj namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 38567387-2772-4b34-b16f-259964e04a2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d0081-685667769f-jhtmj to ip-10-0-141-213.ec2.internal metadata: creationTimestamp: "2026-07-01T17:28:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:28:46Z" name: switch-graph-d0081-685667769f-jhtmj.18be3a97fa491f5f namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: b5d9624e-11ca-42f8-aa79-499cc988c8f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-67b4f8955b-n229x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:47Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d0081-685667769f-jhtmj namespace: kserve-ci-e2e-test resourceVersion: "15906" uid: 38567387-2772-4b34-b16f-259964e04a2f kind: Event lastTimestamp: "2026-07-01T17:28:47Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:28:47Z" name: switch-graph-d0081-685667769f-jhtmj.18be3a981341eb34 namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: ea90e0bf-a56e-4f44-8652-af67000d62fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0081} kind: Pod name: switch-graph-d0081-685667769f-jhtmj namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 38567387-2772-4b34-b16f-259964e04a2f kind: Event lastTimestamp: "2026-07-01T17:28:47Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" metadata: creationTimestamp: "2026-07-01T17:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:28:47Z" name: switch-graph-d0081-685667769f-jhtmj.18be3a9814860804 namespace: kserve-ci-e2e-test resourceVersion: "15919" uid: cf1b4555-7935-432a-9587-e122d9a64b23 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0081} kind: Pod name: switch-graph-d0081-685667769f-jhtmj namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 38567387-2772-4b34-b16f-259964e04a2f kind: Event lastTimestamp: "2026-07-01T17:28:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:e8a5820147018050898833b8b880f51001edf7bc325f6768e16c9212ddb97948" in 2.91s (2.91s including waiting). Image size: 218070135 bytes.' metadata: creationTimestamp: "2026-07-01T17:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:28:50Z" name: switch-graph-d0081-685667769f-jhtmj.18be3a98c1fe9896 namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 42bc6892-7156-433d-8a92-b8001f3594a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0081} kind: Pod name: switch-graph-d0081-685667769f-jhtmj namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 38567387-2772-4b34-b16f-259964e04a2f kind: Event lastTimestamp: "2026-07-01T17:28:50Z" message: 'Created container: switch-graph-d0081' metadata: creationTimestamp: "2026-07-01T17:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:28:50Z" name: switch-graph-d0081-685667769f-jhtmj.18be3a98c4d5f5a4 namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: 3b4f6148-ccb6-4965-b90c-b02a96779cf3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0081} kind: Pod name: switch-graph-d0081-685667769f-jhtmj namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 38567387-2772-4b34-b16f-259964e04a2f kind: Event lastTimestamp: "2026-07-01T17:28:50Z" message: Started container switch-graph-d0081 metadata: creationTimestamp: "2026-07-01T17:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:28:50Z" name: switch-graph-d0081-685667769f-jhtmj.18be3a98c53d6eef namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 0d30b4aa-d60e-47ec-b908-71ffd7069768 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0081} kind: Pod name: switch-graph-d0081-685667769f-jhtmj namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 38567387-2772-4b34-b16f-259964e04a2f kind: Event lastTimestamp: "2026-07-01T17:29:01Z" message: Stopping container switch-graph-d0081 metadata: creationTimestamp: "2026-07-01T17:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:01Z" name: switch-graph-d0081-685667769f-jhtmj.18be3a9b43fa80f1 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: fb815dad-ff13-4086-b4b4-bf3d55d95228 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:29:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0081} kind: Pod name: switch-graph-d0081-685667769f-jhtmj namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 38567387-2772-4b34-b16f-259964e04a2f kind: Event lastTimestamp: "2026-07-01T17:29:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:27Z" name: switch-graph-d0081-685667769f-jhtmj.18be3a9b7e993620 namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: 9fb65389-01c1-4482-b45a-85a69c4cf1f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-213.ec2.internal source: component: kubelet host: ip-10-0-141-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d0081-685667769f namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 79614841-d326-4db5-bca2-085bcc47999d kind: Event lastTimestamp: "2026-07-01T17:28:46Z" message: 'Created pod: switch-graph-d0081-685667769f-jhtmj' metadata: creationTimestamp: "2026-07-01T17:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:46Z" name: switch-graph-d0081-685667769f.18be3a97f9dc5bbf namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: f867997b-12f7-4a2e-87ab-9a62df40bd63 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d0081 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 4d4c0fe3-4a7d-4594-ab12-d39a3601e0ca kind: Event lastTimestamp: "2026-07-01T17:28:46Z" message: Scaled up replica set switch-graph-d0081-685667769f from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:28:46Z" name: switch-graph-d0081.18be3a97f8f140a8 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 898a0688-a107-4f1d-9ab0-7bee25d01991 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:28:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d0081 namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: d133999e-3a65-4a80-be4a-e065de60c0eb kind: Event lastTimestamp: "2026-07-01T17:28:46Z" message: InferenceGraph [switch-graph-d0081] is Ready metadata: creationTimestamp: "2026-07-01T17:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:28:46Z" name: switch-graph-d0081.18be3a97fe7f95a0 namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 3c760cee-9a39-4317-bdd6-b96d0c671d40 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "34613"