--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:57:26.771686Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3ffea-cfb579db8-9spn7 namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: 30aae997-54b9-46f6-8ff7-17f498f7bc77 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-3ffea-cfb579db8-9spn7 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:57:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:57:26Z" name: ensemble-graph-3ffea-cfb579db8-9spn7.18a90ad70fae2c9f namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 2a5317dc-90fd-495b-ab23-c8a3b07b6d45 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3ffea-cfb579db8-9spn7 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 30aae997-54b9-46f6-8ff7-17f498f7bc77 kind: Event lastTimestamp: "2026-04-23T16:57:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-3ffea-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T16:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:57:26Z" name: ensemble-graph-3ffea-cfb579db8-9spn7.18a90ad71ac7c9ec namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: b17d3a94-07a5-4842-a5cd-82e24e63ca1c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:27Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3ffea-cfb579db8-9spn7 namespace: kserve-ci-e2e-test resourceVersion: "20894" uid: 30aae997-54b9-46f6-8ff7-17f498f7bc77 kind: Event lastTimestamp: "2026-04-23T16:57:27Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:57:27Z" name: ensemble-graph-3ffea-cfb579db8-9spn7.18a90ad74c8292f3 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: 2ffd7a60-7c72-42fc-b7b7-7caac233058a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3ffea} kind: Pod name: ensemble-graph-3ffea-cfb579db8-9spn7 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 30aae997-54b9-46f6-8ff7-17f498f7bc77 kind: Event lastTimestamp: "2026-04-23T16:57:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:57:27Z" name: ensemble-graph-3ffea-cfb579db8-9spn7.18a90ad74dbd4343 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 8ad848ff-658c-41c5-b764-0fccf369e340 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3ffea} kind: Pod name: ensemble-graph-3ffea-cfb579db8-9spn7 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 30aae997-54b9-46f6-8ff7-17f498f7bc77 kind: Event lastTimestamp: "2026-04-23T16:57:27Z" message: 'Created container: ensemble-graph-3ffea' metadata: creationTimestamp: "2026-04-23T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:57:27Z" name: ensemble-graph-3ffea-cfb579db8-9spn7.18a90ad7504ce7f2 namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 2eee0650-5c31-46e9-ad12-099aec63948b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3ffea} kind: Pod name: ensemble-graph-3ffea-cfb579db8-9spn7 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 30aae997-54b9-46f6-8ff7-17f498f7bc77 kind: Event lastTimestamp: "2026-04-23T16:57:27Z" message: Started container ensemble-graph-3ffea metadata: creationTimestamp: "2026-04-23T16:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:57:27Z" name: ensemble-graph-3ffea-cfb579db8-9spn7.18a90ad750b8ea7f namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: e0660175-1b9d-4efa-8b9f-b2974af82b1b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3ffea} kind: Pod name: ensemble-graph-3ffea-cfb579db8-9spn7 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 30aae997-54b9-46f6-8ff7-17f498f7bc77 kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Stopping container ensemble-graph-3ffea metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:05:41Z" name: ensemble-graph-3ffea-cfb579db8-9spn7.18a90b4a3cf2336a namespace: kserve-ci-e2e-test resourceVersion: "24392" uid: f16e3211-3011-4421-b250-a03fd57a0781 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3ffea} kind: Pod name: ensemble-graph-3ffea-cfb579db8-9spn7 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 30aae997-54b9-46f6-8ff7-17f498f7bc77 kind: Event lastTimestamp: "2026-04-23T17:06:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T17:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:09Z" name: ensemble-graph-3ffea-cfb579db8-9spn7.18a90b4afb099b9e namespace: kserve-ci-e2e-test resourceVersion: "24715" uid: db181d3d-9922-4e8e-98e2-f26dbd25b553 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-3ffea-cfb579db8 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 6684ab7d-09d9-41ab-85f6-d11540d84546 kind: Event lastTimestamp: "2026-04-23T16:57:26Z" message: 'Created pod: ensemble-graph-3ffea-cfb579db8-9spn7' metadata: creationTimestamp: "2026-04-23T16:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:26Z" name: ensemble-graph-3ffea-cfb579db8.18a90ad70f1513bd namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 68e1a4b9-bd9f-4500-b223-09cf1ad08997 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-3ffea namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 4b732013-eb0c-41dc-8515-ddbcce1b95f5 kind: Event lastTimestamp: "2026-04-23T16:57:26Z" message: Scaled up replica set ensemble-graph-3ffea-cfb579db8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:26Z" name: ensemble-graph-3ffea.18a90ad70e3bc583 namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: e3eaf4eb-90d6-4484-9dd3-377f0c651515 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-3ffea namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 428a1216-947d-4839-8f42-490fa005f4ce kind: Event lastTimestamp: "2026-04-23T16:57:26Z" message: InferenceGraph [ensemble-graph-3ffea] is Ready metadata: creationTimestamp: "2026-04-23T16:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:57:26Z" name: ensemble-graph-3ffea.18a90ad71320b9f2 namespace: kserve-ci-e2e-test resourceVersion: "20899" uid: 8c7c4386-0f07-40f6-8023-20c1b357c4ec reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:57:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3ffea namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 1e414f2b-946c-4485-b867-736a82a9cdb3 kind: Event lastTimestamp: "2026-04-23T16:57:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:56Z" name: ensemble-graph-3ffea.18a90ada8cc950ee namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: d81e9911-1039-4013-8928-ad7b11d6e6f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:57:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3ffea namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 1e414f2b-946c-4485-b867-736a82a9cdb3 kind: Event lastTimestamp: "2026-04-23T16:57:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:56Z" name: ensemble-graph-3ffea.18a90ada8d181e0f namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: 18bf5994-f628-4a20-a9fc-39a0dea936eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:56:06.482865Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c88c1-79c8484c9b-hmj55 namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 21a3ee2f-93ab-4870-901f-2efa216407ac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c88c1-79c8484c9b-hmj55 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:56:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:56:06Z" name: ensemble-graph-c88c1-79c8484c9b-hmj55.18a90ac45e18076e namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: 0cf075ef-c18b-4f54-85ec-d9fef22c6b65 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:56:06Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c88c1-79c8484c9b-hmj55 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 21a3ee2f-93ab-4870-901f-2efa216407ac kind: Event lastTimestamp: "2026-04-23T16:56:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-c88c1-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T16:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:07Z" name: ensemble-graph-c88c1-79c8484c9b-hmj55.18a90ac46b6aa0d0 namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 4f2cabab-a73b-4b72-b5a2-ce25976ccdb6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:08Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c88c1-79c8484c9b-hmj55 namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 21a3ee2f-93ab-4870-901f-2efa216407ac kind: Event lastTimestamp: "2026-04-23T16:56:08Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:56:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:56:08Z" name: ensemble-graph-c88c1-79c8484c9b-hmj55.18a90ac4d0669140 namespace: kserve-ci-e2e-test resourceVersion: "19895" uid: bca80ec9-b584-425a-beb8-1e9f953bcafc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c88c1} kind: Pod name: ensemble-graph-c88c1-79c8484c9b-hmj55 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 21a3ee2f-93ab-4870-901f-2efa216407ac kind: Event lastTimestamp: "2026-04-23T16:56:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T16:56:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:08Z" name: ensemble-graph-c88c1-79c8484c9b-hmj55.18a90ac4d18e1e21 namespace: kserve-ci-e2e-test resourceVersion: "19898" uid: 827cab3b-a34e-4f00-a3bb-eae010d8ff6a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c88c1} kind: Pod name: ensemble-graph-c88c1-79c8484c9b-hmj55 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 21a3ee2f-93ab-4870-901f-2efa216407ac kind: Event lastTimestamp: "2026-04-23T16:56:08Z" message: 'Created container: ensemble-graph-c88c1' metadata: creationTimestamp: "2026-04-23T16:56:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:08Z" name: ensemble-graph-c88c1-79c8484c9b-hmj55.18a90ac4d44bf9f3 namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: 8bc2b608-454d-4edc-a11d-584601417c40 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c88c1} kind: Pod name: ensemble-graph-c88c1-79c8484c9b-hmj55 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 21a3ee2f-93ab-4870-901f-2efa216407ac kind: Event lastTimestamp: "2026-04-23T16:56:08Z" message: Started container ensemble-graph-c88c1 metadata: creationTimestamp: "2026-04-23T16:56:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:08Z" name: ensemble-graph-c88c1-79c8484c9b-hmj55.18a90ac4d4b811e5 namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: d0430c13-c7dd-43fb-bc32-7e679bf8ebe8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c88c1} kind: Pod name: ensemble-graph-c88c1-79c8484c9b-hmj55 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 21a3ee2f-93ab-4870-901f-2efa216407ac kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Stopping container ensemble-graph-c88c1 metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:16Z" name: ensemble-graph-c88c1-79c8484c9b-hmj55.18a90ac6b6cb9659 namespace: kserve-ci-e2e-test resourceVersion: "19985" uid: 239304ee-75fd-4251-a404-ef2d169e04ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c88c1} kind: Pod name: ensemble-graph-c88c1-79c8484c9b-hmj55 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 21a3ee2f-93ab-4870-901f-2efa216407ac kind: Event lastTimestamp: "2026-04-23T16:56:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T16:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:45Z" name: ensemble-graph-c88c1-79c8484c9b-hmj55.18a90ac79c9174ba namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 2baf3cd7-ba69-45d6-86f8-03309b229fde reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c88c1-79c8484c9b namespace: kserve-ci-e2e-test resourceVersion: "19859" uid: 4d01ff2f-a1a2-4b15-93cc-7d094e63bb17 kind: Event lastTimestamp: "2026-04-23T16:56:06Z" message: 'Created pod: ensemble-graph-c88c1-79c8484c9b-hmj55' metadata: creationTimestamp: "2026-04-23T16:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:06Z" name: ensemble-graph-c88c1-79c8484c9b.18a90ac45dbf9c15 namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 2d519670-4415-45c6-bde2-139e74cab1dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c88c1 namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: 48ff0e43-4e49-43a8-bd99-3b58ad8bf86b kind: Event lastTimestamp: "2026-04-23T16:56:06Z" message: Scaled up replica set ensemble-graph-c88c1-79c8484c9b from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:06Z" name: ensemble-graph-c88c1.18a90ac45d05556d namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: 33d6e635-abe8-4d53-b106-dbf39f96b4d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c88c1 namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: 80360b5e-a62b-49e3-8207-b2bec28e81d7 kind: Event lastTimestamp: "2026-04-23T16:56:06Z" message: InferenceGraph [ensemble-graph-c88c1] is Ready metadata: creationTimestamp: "2026-04-23T16:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:06Z" name: ensemble-graph-c88c1.18a90ac460401f31 namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: 38b07d59-64c5-4c07-8f7d-31003630759c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T17:06:21.421902Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9 namespace: kserve-ci-e2e-test resourceVersion: "24881" uid: f9438b07-56b1-4b01-bab3-1e245eba99ac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9 to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T17:06:21Z" name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9.18a90b538b511d00 namespace: kserve-ci-e2e-test resourceVersion: "24888" uid: fc783554-db03-4c2f-85ec-2e7addc945e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9 namespace: kserve-ci-e2e-test resourceVersion: "24895" uid: f9438b07-56b1-4b01-bab3-1e245eba99ac kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T17:06:21Z" name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9.18a90b5393974c64 namespace: kserve-ci-e2e-test resourceVersion: "24907" uid: 60a846c0-6544-46fe-b7ec-acbc3d1eef0a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9 namespace: kserve-ci-e2e-test resourceVersion: "24884" uid: f9438b07-56b1-4b01-bab3-1e245eba99ac kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:21Z" name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9.18a90b5394cc0208 namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: e1947634-154d-407c-b28a-81e395871727 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9 namespace: kserve-ci-e2e-test resourceVersion: "24884" uid: f9438b07-56b1-4b01-bab3-1e245eba99ac kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:21Z" name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9.18a90b5399331545 namespace: kserve-ci-e2e-test resourceVersion: "24912" uid: 692fa773-14e7-48fc-a27d-691a9e7b3952 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9 namespace: kserve-ci-e2e-test resourceVersion: "24884" uid: f9438b07-56b1-4b01-bab3-1e245eba99ac kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:21Z" name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9.18a90b5399a8af8e namespace: kserve-ci-e2e-test resourceVersion: "24913" uid: d07e7e22-0068-4dc0-a315-309c688ba388 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T17:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9 namespace: kserve-ci-e2e-test resourceVersion: "24884" uid: f9438b07-56b1-4b01-bab3-1e245eba99ac kind: Event lastTimestamp: "2026-04-23T17:23:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T17:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:23:53Z" name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9.18a90b53aec68629 namespace: kserve-ci-e2e-test resourceVersion: "32973" uid: c037e62c-8d86-4649-a890-f134595fd76f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9 namespace: kserve-ci-e2e-test resourceVersion: "24884" uid: f9438b07-56b1-4b01-bab3-1e245eba99ac kind: Event lastTimestamp: "2026-04-23T17:23:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:23:50Z" name: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9.18a90c47dfe00191 namespace: kserve-ci-e2e-test resourceVersion: "32926" uid: 7fd6f8f8-0778-4c41-b37a-53512e1fb5a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-390fe-predictor-5d9d5697cb namespace: kserve-ci-e2e-test resourceVersion: "24865" uid: ac01dca4-fb32-4360-a364-8d0c00495b2d kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: 'Created pod: error-404-isvc-390fe-predictor-5d9d5697cb-ct4l9' metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:21Z" name: error-404-isvc-390fe-predictor-5d9d5697cb.18a90b538aaf839b namespace: kserve-ci-e2e-test resourceVersion: "24883" uid: cf8b374f-2ddd-447d-9ee2-590d7731a3a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "24862" uid: 7251d700-e2f0-4d5f-9754-8b5e160b9cf5 kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Scaled up replica set error-404-isvc-390fe-predictor-5d9d5697cb from 0 to 1 metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:21Z" name: error-404-isvc-390fe-predictor.18a90b5389045c6f namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: a5a29b2e-fdb9-45d8-bfc2-c5d3e02932ac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T17:06:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "24864" uid: 25c892b4-ebf0-48c0-96b3-c939df6aac78 kind: Event lastTimestamp: "2026-04-23T17:07:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:06Z" name: error-404-isvc-390fe-predictor.18a90b5707a165d7 namespace: kserve-ci-e2e-test resourceVersion: "25478" uid: dcdb1015-d295-4162-a397-94c6d32efc98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T17:06:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "24864" uid: 25c892b4-ebf0-48c0-96b3-c939df6aac78 kind: Event lastTimestamp: "2026-04-23T17:07:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:06Z" name: error-404-isvc-390fe-predictor.18a90b570805eb62 namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: d623543e-fd71-46da-8bae-939d71d47512 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:07:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "25073" uid: 25c892b4-ebf0-48c0-96b3-c939df6aac78 kind: Event lastTimestamp: "2026-04-23T17:07:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:36Z" name: error-404-isvc-390fe-predictor.18a90b61845ea424 namespace: kserve-ci-e2e-test resourceVersion: "25861" uid: 94aa8b32-a876-4586-8c20-06f2806cf841 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:07:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "25073" uid: 25c892b4-ebf0-48c0-96b3-c939df6aac78 kind: Event lastTimestamp: "2026-04-23T17:07:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:36Z" name: error-404-isvc-390fe-predictor.18a90b6184c1c925 namespace: kserve-ci-e2e-test resourceVersion: "25862" uid: ae599209-afd1-4d3a-9ada-3014772413da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390fe namespace: kserve-ci-e2e-test resourceVersion: "24848" uid: 477dbcb9-c051-4bce-aaec-666e67311322 kind: Event lastTimestamp: "2026-04-23T17:07:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:13Z" name: error-404-isvc-390fe.18a90b5388fb0679 namespace: kserve-ci-e2e-test resourceVersion: "25540" uid: 07771703-6672-4d6e-bd00-7de65a7a041e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390fe namespace: kserve-ci-e2e-test resourceVersion: "25535" uid: 477dbcb9-c051-4bce-aaec-666e67311322 kind: Event lastTimestamp: "2026-04-23T17:07:13Z" message: InferenceService [error-404-isvc-390fe] is Ready metadata: creationTimestamp: "2026-04-23T17:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:13Z" name: error-404-isvc-390fe.18a90b5f9449048f namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: ca845e47-6812-49be-b3c2-19d74a612125 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:46:11.410851Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-39400-predictor-755bc7b66-hcrwc namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 027ca991-c02f-4657-9ff8-fbc4abde661c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-39400-predictor-755bc7b66-hcrwc to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:46:11Z" name: error-404-isvc-39400-predictor-755bc7b66-hcrwc.18a90a39d109b297 namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 08c10301-e72c-46c5-abdb-100b9e63118f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-39400-predictor-755bc7b66-hcrwc namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: 027ca991-c02f-4657-9ff8-fbc4abde661c kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:46:11Z" name: error-404-isvc-39400-predictor-755bc7b66-hcrwc.18a90a39d8886100 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: fac9b56c-472e-454d-9478-a1b27e6a9f71 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39400-predictor-755bc7b66-hcrwc namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 027ca991-c02f-4657-9ff8-fbc4abde661c kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: error-404-isvc-39400-predictor-755bc7b66-hcrwc.18a90a39d9bd1453 namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: 0b2a7cd7-02c7-4945-b469-39edd448a476 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39400-predictor-755bc7b66-hcrwc namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 027ca991-c02f-4657-9ff8-fbc4abde661c kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: error-404-isvc-39400-predictor-755bc7b66-hcrwc.18a90a39dfa10185 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: 52a58edd-cb88-4fb4-9864-3df605c75dda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39400-predictor-755bc7b66-hcrwc namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 027ca991-c02f-4657-9ff8-fbc4abde661c kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: error-404-isvc-39400-predictor-755bc7b66-hcrwc.18a90a39e01b0b29 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 91d8d12a-7c17-4981-811f-f463f46c537e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39400-predictor-755bc7b66-hcrwc namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 027ca991-c02f-4657-9ff8-fbc4abde661c kind: Event lastTimestamp: "2026-04-23T16:46:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:52Z" name: error-404-isvc-39400-predictor-755bc7b66-hcrwc.18a90a39ec476dab namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 095da5a3-d46b-447b-8c0d-42b49643f6c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39400-predictor-755bc7b66-hcrwc namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 027ca991-c02f-4657-9ff8-fbc4abde661c kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-39400-predictor-755bc7b66-hcrwc.18a90abd568044f2 namespace: kserve-ci-e2e-test resourceVersion: "19544" uid: 5ec07f16-1b6c-4a6b-9e13-0c528258ad17 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-39400-predictor-755bc7b66 namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: e2a89fb9-4fa3-4cf5-9398-69c2dee01514 kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: 'Created pod: error-404-isvc-39400-predictor-755bc7b66-hcrwc' metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:11Z" name: error-404-isvc-39400-predictor-755bc7b66.18a90a39d0721c65 namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: 10076ea8-ac3e-494e-a51d-36170dc93ad3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-39400-predictor namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 6a57c18b-2ffa-45ac-a93a-19c608e4ff89 kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Scaled up replica set error-404-isvc-39400-predictor-755bc7b66 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:11Z" name: error-404-isvc-39400-predictor.18a90a39cef9adeb namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: 76922719-ae6a-454b-bac6-3adaca72f049 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-39400-predictor namespace: kserve-ci-e2e-test resourceVersion: "15196" uid: d77d30c1-6e15-43e8-8035-03f384df16ff kind: Event lastTimestamp: "2026-04-23T16:46:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:26Z" name: error-404-isvc-39400-predictor.18a90a3d4d76d2ba namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 2056a949-8dba-4595-adce-aca983301fb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-39400-predictor namespace: kserve-ci-e2e-test resourceVersion: "15196" uid: d77d30c1-6e15-43e8-8035-03f384df16ff kind: Event lastTimestamp: "2026-04-23T16:46:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:26Z" name: error-404-isvc-39400-predictor.18a90a3d4de0205a namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 6e6d91f5-7fe4-47b7-9412-35711d5e03c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-39400-predictor namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: d77d30c1-6e15-43e8-8035-03f384df16ff kind: Event lastTimestamp: "2026-04-23T16:47:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:47:26Z" name: error-404-isvc-39400-predictor.18a90a40ccee822f namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: bfdaf4c4-8d58-4961-84f5-a866b6d88464 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-39400-predictor namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: d77d30c1-6e15-43e8-8035-03f384df16ff kind: Event lastTimestamp: "2026-04-23T16:47:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:47:26Z" name: error-404-isvc-39400-predictor.18a90a40cd2de4d2 namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 62105513-da2a-4a61-8d1a-b23f973a59f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-39400 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 3eea400a-693e-4036-9850-35159bf21daf kind: Event lastTimestamp: "2026-04-23T16:47:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:47:02Z" name: error-404-isvc-39400.18a90a39ced1468e namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: 594a86e5-fa0d-406c-81ae-fff41c292ef1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-39400 namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 3eea400a-693e-4036-9850-35159bf21daf kind: Event lastTimestamp: "2026-04-23T16:47:02Z" message: InferenceService [error-404-isvc-39400] is Ready metadata: creationTimestamp: "2026-04-23T16:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:47:02Z" name: error-404-isvc-39400.18a90a45cfbccce5 namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: 5d8b3be0-ccb4-494e-b648-f74c11a37540 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:56:16.797611Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: bcf88044-c1a4-40b6-b6ff-4fe41d23c6ad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:56:16Z" name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh.18a90ac6c4e68895 namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: 3221fdea-f28e-4577-bcba-c12b5152fa30 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: bcf88044-c1a4-40b6-b6ff-4fe41d23c6ad kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:56:16Z" name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh.18a90ac6cca11417 namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: daa455c1-4740-4510-805a-cd112a4a6974 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: bcf88044-c1a4-40b6-b6ff-4fe41d23c6ad kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:16Z" name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh.18a90ac6cde4d104 namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 143a9ed4-fe13-4c0e-a233-b604676c4351 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: bcf88044-c1a4-40b6-b6ff-4fe41d23c6ad kind: Event lastTimestamp: "2026-04-23T16:56:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:17Z" name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh.18a90ac6d444e96b namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: 4afe064d-8fe8-4678-9cd2-8f9ceb8e4ef6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: bcf88044-c1a4-40b6-b6ff-4fe41d23c6ad kind: Event lastTimestamp: "2026-04-23T16:56:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:17Z" name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh.18a90ac6d4c0ac1e namespace: kserve-ci-e2e-test resourceVersion: "20080" uid: dc2d97e1-8472-447f-8ad5-2e34489340e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:56:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: bcf88044-c1a4-40b6-b6ff-4fe41d23c6ad kind: Event lastTimestamp: "2026-04-23T16:56:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:58Z" name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh.18a90ac7089321cd namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: 6675dc47-1737-46f2-b7aa-b1da68de9ee0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: bcf88044-c1a4-40b6-b6ff-4fe41d23c6ad kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh.18a90b4a460e5604 namespace: kserve-ci-e2e-test resourceVersion: "24448" uid: f4fe5c9b-c8a2-411d-bb28-b686ea1adf79 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3ffea-predictor-c9bf7cd76 namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: 9db87950-0917-43f4-88d9-e0aaf9fe5c74 kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: 'Created pod: error-404-isvc-3ffea-predictor-c9bf7cd76-kpqnh' metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:16Z" name: error-404-isvc-3ffea-predictor-c9bf7cd76.18a90ac6c4771436 namespace: kserve-ci-e2e-test resourceVersion: "20053" uid: 7c240a9a-9ca9-43eb-887a-765a312fa819 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: e9d9a9ca-f907-4651-a46c-2a11d16f12eb kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Scaled up replica set error-404-isvc-3ffea-predictor-c9bf7cd76 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:16Z" name: error-404-isvc-3ffea-predictor.18a90ac6c308b4ad namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: 62fe70f4-b96e-44a0-8c4e-9b8f1953d15e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 7bd37581-f316-4e08-8f2c-cbdf226993c1 kind: Event lastTimestamp: "2026-04-23T16:56:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:31Z" name: error-404-isvc-3ffea-predictor.18a90aca41923b37 namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: e866172b-11cd-49b6-b4f0-763377f3c4a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 7bd37581-f316-4e08-8f2c-cbdf226993c1 kind: Event lastTimestamp: "2026-04-23T16:56:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:31Z" name: error-404-isvc-3ffea-predictor.18a90aca41fcfe3e namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: fe6e1fb5-90e7-40d8-9787-f0bd403a8864 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:56:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 7bd37581-f316-4e08-8f2c-cbdf226993c1 kind: Event lastTimestamp: "2026-04-23T16:57:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:31Z" name: error-404-isvc-3ffea-predictor.18a90acdc0a2fe1b namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: 538bc55b-3fcc-4eef-9152-ec4a9190628f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:56:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 7bd37581-f316-4e08-8f2c-cbdf226993c1 kind: Event lastTimestamp: "2026-04-23T16:57:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:31Z" name: error-404-isvc-3ffea-predictor.18a90acdc103f9be namespace: kserve-ci-e2e-test resourceVersion: "20950" uid: 9472853a-ccd7-4401-8d2f-a02e1a58d5fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ffea namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: b9ff1a9d-c72d-4a9e-9a7b-2217806cdec8 kind: Event lastTimestamp: "2026-04-23T16:57:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:57:09Z" name: error-404-isvc-3ffea.18a90ac6c2e52a26 namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 9dc667b3-0ec5-476b-b991-f481339c69df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ffea namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: b9ff1a9d-c72d-4a9e-9a7b-2217806cdec8 kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-3ffea": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3ffea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:16Z" name: error-404-isvc-3ffea.18a90ac6c3336f42 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: d2d02f02-c4d0-40ca-be96-082ef3f53f7a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ffea namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: b9ff1a9d-c72d-4a9e-9a7b-2217806cdec8 kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3ffea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:16Z" name: error-404-isvc-3ffea.18a90ac6c333d119 namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: 22ec9bf0-15f7-46a4-b092-a2015c931a71 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ffea namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: b9ff1a9d-c72d-4a9e-9a7b-2217806cdec8 kind: Event lastTimestamp: "2026-04-23T16:57:08Z" message: InferenceService [error-404-isvc-3ffea] is Ready metadata: creationTimestamp: "2026-04-23T16:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:57:08Z" name: error-404-isvc-3ffea.18a90ad2ec10e3ac namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: d8d9e141-fe06-4fd2-b8aa-8dbc50aa741f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T17:05:41.643658Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz namespace: kserve-ci-e2e-test resourceVersion: "24461" uid: d7a2aa2a-1d34-47c8-87ad-ce6f739ee4ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-56812-predictor-7f65bbd66b-64cmz to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz.18a90b4a48594819 namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: e7423070-366f-4c5a-9af5-64bcda31fa93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz namespace: kserve-ci-e2e-test resourceVersion: "24472" uid: d7a2aa2a-1d34-47c8-87ad-ce6f739ee4ca kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz.18a90b4a4faaf620 namespace: kserve-ci-e2e-test resourceVersion: "24480" uid: 9102d5b1-458a-437b-a77e-8745c7e2a8e5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz namespace: kserve-ci-e2e-test resourceVersion: "24464" uid: d7a2aa2a-1d34-47c8-87ad-ce6f739ee4ca kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz.18a90b4a511efed6 namespace: kserve-ci-e2e-test resourceVersion: "24486" uid: ae739b24-9034-4280-bf5c-ade56ad5b5fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz namespace: kserve-ci-e2e-test resourceVersion: "24464" uid: d7a2aa2a-1d34-47c8-87ad-ce6f739ee4ca kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz.18a90b4a54eae94e namespace: kserve-ci-e2e-test resourceVersion: "24490" uid: f1919f62-fe8e-40b8-9580-fdbcd2619810 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz namespace: kserve-ci-e2e-test resourceVersion: "24464" uid: d7a2aa2a-1d34-47c8-87ad-ce6f739ee4ca kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz.18a90b4a557595ba namespace: kserve-ci-e2e-test resourceVersion: "24491" uid: 01bf199a-635a-4b11-83ee-5b137704a94e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz namespace: kserve-ci-e2e-test resourceVersion: "24464" uid: d7a2aa2a-1d34-47c8-87ad-ce6f739ee4ca kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:02Z" name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz.18a90b4a567e2187 namespace: kserve-ci-e2e-test resourceVersion: "25435" uid: b1af0d96-6163-4980-81cc-551f4d6e95d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz namespace: kserve-ci-e2e-test resourceVersion: "24464" uid: d7a2aa2a-1d34-47c8-87ad-ce6f739ee4ca kind: Event lastTimestamp: "2026-04-23T17:07:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:01Z" name: error-404-isvc-56812-predictor-7f65bbd66b-64cmz.18a90b5cf4d9c351 namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: a9e954e6-670d-4cd3-9417-8addd1dea078 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-56812-predictor-7f65bbd66b namespace: kserve-ci-e2e-test resourceVersion: "24451" uid: ec92b4e2-7189-4b09-b2de-5cd6ab82b1ad kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: 'Created pod: error-404-isvc-56812-predictor-7f65bbd66b-64cmz' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-56812-predictor-7f65bbd66b.18a90b4a47f64ea7 namespace: kserve-ci-e2e-test resourceVersion: "24463" uid: 789e6d71-6994-4d32-8731-cddf51c45202 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24446" uid: 2da68a9f-4810-480a-bd8e-59364e3edb2f kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Scaled up replica set error-404-isvc-56812-predictor-7f65bbd66b from 0 to 1 metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-56812-predictor.18a90b4a4692ff9d namespace: kserve-ci-e2e-test resourceVersion: "24455" uid: e3250b10-5352-4bfa-810b-a40ee23f736c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24450" uid: 982a049d-cedb-4a0f-852a-9667c8a91e27 kind: Event lastTimestamp: "2026-04-23T17:05:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:05:56Z" name: error-404-isvc-56812-predictor.18a90b4dc53a50a9 namespace: kserve-ci-e2e-test resourceVersion: "24617" uid: ef4c22f9-ae45-4e02-b80b-1b08ef48ac50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24450" uid: 982a049d-cedb-4a0f-852a-9667c8a91e27 kind: Event lastTimestamp: "2026-04-23T17:05:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:05:56Z" name: error-404-isvc-56812-predictor.18a90b4dc59e3710 namespace: kserve-ci-e2e-test resourceVersion: "24619" uid: 13a88569-b408-44ac-959a-749e339db0f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T17:06:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 982a049d-cedb-4a0f-852a-9667c8a91e27 kind: Event lastTimestamp: "2026-04-23T17:06:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:56Z" name: error-404-isvc-56812-predictor.18a90b51444262f0 namespace: kserve-ci-e2e-test resourceVersion: "25268" uid: b689797e-cd23-4a3f-91fb-3a1f37b0f477 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T17:06:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24618" uid: 982a049d-cedb-4a0f-852a-9667c8a91e27 kind: Event lastTimestamp: "2026-04-23T17:06:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:56Z" name: error-404-isvc-56812-predictor.18a90b51449ee806 namespace: kserve-ci-e2e-test resourceVersion: "25269" uid: c9fa926b-26e2-4a78-9990-0f87c2d748d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-56812 namespace: kserve-ci-e2e-test resourceVersion: "24424" uid: b112fd9a-d526-41d4-9613-99b4616547cd kind: Event lastTimestamp: "2026-04-23T17:06:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:06:33Z" name: error-404-isvc-56812.18a90b4a46995ead namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: 45124534-16a6-4122-825c-47ef1d254ac6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-56812 namespace: kserve-ci-e2e-test resourceVersion: "24424" uid: b112fd9a-d526-41d4-9613-99b4616547cd kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-56812": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-56812": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-56812.18a90b4a47410b98 namespace: kserve-ci-e2e-test resourceVersion: "24459" uid: f46e04ff-ee77-47c3-abe0-a2e39fd1b66c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-56812 namespace: kserve-ci-e2e-test resourceVersion: "24424" uid: b112fd9a-d526-41d4-9613-99b4616547cd kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-56812": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:05:41Z" name: error-404-isvc-56812.18a90b4a47414788 namespace: kserve-ci-e2e-test resourceVersion: "24462" uid: 73d80013-f4a8-4231-ac7e-1ac30e9c044b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-56812 namespace: kserve-ci-e2e-test resourceVersion: "25032" uid: b112fd9a-d526-41d4-9613-99b4616547cd kind: Event lastTimestamp: "2026-04-23T17:06:32Z" message: InferenceService [error-404-isvc-56812] is Ready metadata: creationTimestamp: "2026-04-23T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:06:32Z" name: error-404-isvc-56812.18a90b563a7494db namespace: kserve-ci-e2e-test resourceVersion: "25033" uid: 788f84be-3f88-49db-a65f-7cedbd77b336 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T17:07:01.972531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6 namespace: kserve-ci-e2e-test resourceVersion: "25393" uid: c8431775-fe3c-414b-9d9b-e9586527e351 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6 to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T17:07:01Z" name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6.18a90b5cfc529570 namespace: kserve-ci-e2e-test resourceVersion: "25399" uid: bd4da7a3-e409-4035-8161-0fa9da6add01 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6 namespace: kserve-ci-e2e-test resourceVersion: "25402" uid: c8431775-fe3c-414b-9d9b-e9586527e351 kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T17:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T17:07:02Z" name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6.18a90b5d041c554f namespace: kserve-ci-e2e-test resourceVersion: "25414" uid: 8b98bb74-8213-447d-92de-e9fcf7eb64ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6 namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: c8431775-fe3c-414b-9d9b-e9586527e351 kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T17:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:02Z" name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6.18a90b5d055c8e29 namespace: kserve-ci-e2e-test resourceVersion: "25418" uid: 7ab0b2b8-e765-47e4-8267-25586367f3ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6 namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: c8431775-fe3c-414b-9d9b-e9586527e351 kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T17:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:02Z" name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6.18a90b5d0d3bac32 namespace: kserve-ci-e2e-test resourceVersion: "25424" uid: 5b9c2725-1931-4351-b934-b8e65cfb92f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6 namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: c8431775-fe3c-414b-9d9b-e9586527e351 kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T17:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:02Z" name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6.18a90b5d0db63332 namespace: kserve-ci-e2e-test resourceVersion: "25425" uid: 395d1995-ba74-4fe1-90c6-3f3f8fe58aba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:07:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6 namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: c8431775-fe3c-414b-9d9b-e9586527e351 kind: Event lastTimestamp: "2026-04-23T17:07:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T17:07:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:44Z" name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6.18a90b5d4344a2f3 namespace: kserve-ci-e2e-test resourceVersion: "26107" uid: b0aba0ea-694f-47ef-a563-56537a0a2f0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:16:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6 namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: c8431775-fe3c-414b-9d9b-e9586527e351 kind: Event lastTimestamp: "2026-04-23T17:16:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:16:26Z" name: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6.18a90be081ec4c32 namespace: kserve-ci-e2e-test resourceVersion: "29853" uid: 7903f336-f360-4608-96ee-bac37b2af0a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5bc06-predictor-5b55f49dc8 namespace: kserve-ci-e2e-test resourceVersion: "25385" uid: f1357ea8-a452-45c3-95ac-5b8aaa0a83c0 kind: Event lastTimestamp: "2026-04-23T17:07:01Z" message: 'Created pod: error-404-isvc-5bc06-predictor-5b55f49dc8-sxjj6' metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:01Z" name: error-404-isvc-5bc06-predictor-5b55f49dc8.18a90b5cfbec5a3b namespace: kserve-ci-e2e-test resourceVersion: "25394" uid: d0151e38-8ace-4c42-bcaf-8490a4c4d85a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25383" uid: 26077cec-0367-40e9-83d8-0dc398a5d8c8 kind: Event lastTimestamp: "2026-04-23T17:07:01Z" message: Scaled up replica set error-404-isvc-5bc06-predictor-5b55f49dc8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:01Z" name: error-404-isvc-5bc06-predictor.18a90b5cfa6f2efe namespace: kserve-ci-e2e-test resourceVersion: "25388" uid: fa0c2712-c172-4d49-b00a-0d8e29c991a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:07:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25384" uid: 82241748-d49f-448a-a565-76a3ba0a4fd1 kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:31Z" name: error-404-isvc-5bc06-predictor.18a90b6078f25522 namespace: kserve-ci-e2e-test resourceVersion: "25728" uid: a915f538-6c7f-4f34-bf33-6d9727668053 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:07:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25384" uid: 82241748-d49f-448a-a565-76a3ba0a4fd1 kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:31Z" name: error-404-isvc-5bc06-predictor.18a90b60795af26d namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: d6305db4-74ed-4568-81d5-94adb45783e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T17:07:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25576" uid: 82241748-d49f-448a-a565-76a3ba0a4fd1 kind: Event lastTimestamp: "2026-04-23T17:08:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:17Z" name: error-404-isvc-5bc06-predictor.18a90b6776d238c9 namespace: kserve-ci-e2e-test resourceVersion: "26421" uid: bbe25853-3477-4e2c-b97b-3b6099c6e3c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T17:07:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25576" uid: 82241748-d49f-448a-a565-76a3ba0a4fd1 kind: Event lastTimestamp: "2026-04-23T17:08:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:17Z" name: error-404-isvc-5bc06-predictor.18a90b677736c9db namespace: kserve-ci-e2e-test resourceVersion: "26422" uid: 5cf53928-0908-4999-86f5-cf594d05f02b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "25362" uid: ce6e6fd2-7996-4db1-b2bf-9b4a1ea6f854 kind: Event lastTimestamp: "2026-04-23T17:07:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:54Z" name: error-404-isvc-5bc06.18a90b5cfa489b1a namespace: kserve-ci-e2e-test resourceVersion: "26217" uid: 266b8144-a1c1-47f8-9547-619ff56b8038 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "25362" uid: ce6e6fd2-7996-4db1-b2bf-9b4a1ea6f854 kind: Event lastTimestamp: "2026-04-23T17:07:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-5bc06": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5bc06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:01Z" name: error-404-isvc-5bc06.18a90b5cfa9d8eff namespace: kserve-ci-e2e-test resourceVersion: "25390" uid: 9f7218a2-77f9-45cc-97b7-ddb3f3370c3e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "25362" uid: ce6e6fd2-7996-4db1-b2bf-9b4a1ea6f854 kind: Event lastTimestamp: "2026-04-23T17:07:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5bc06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:01Z" name: error-404-isvc-5bc06.18a90b5cfa9dd7b6 namespace: kserve-ci-e2e-test resourceVersion: "25392" uid: e2df0d5f-b428-4007-af74-0e7fc6bf7aab reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: ce6e6fd2-7996-4db1-b2bf-9b4a1ea6f854 kind: Event lastTimestamp: "2026-04-23T17:07:54Z" message: InferenceService [error-404-isvc-5bc06] is Ready metadata: creationTimestamp: "2026-04-23T17:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:54Z" name: error-404-isvc-5bc06.18a90b6926b8fc7f namespace: kserve-ci-e2e-test resourceVersion: "26216" uid: 7e30bf8e-5f02-4462-8fb9-c09d88a6d979 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:44:01.277144Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd namespace: kserve-ci-e2e-test resourceVersion: "13657" uid: 049b30eb-57f1-465f-ab2d-b6f8a18762e9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:44:01Z" name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd.18a90a1b8476dee4 namespace: kserve-ci-e2e-test resourceVersion: "13662" uid: 0f4a41f5-a1cd-4dca-bc2f-05fbd2dbdc85 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd namespace: kserve-ci-e2e-test resourceVersion: "13666" uid: 049b30eb-57f1-465f-ab2d-b6f8a18762e9 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:44:01Z" name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd.18a90a1b8cdf2e09 namespace: kserve-ci-e2e-test resourceVersion: "13704" uid: 960e2928-77cb-4525-b5a7-546d0e66dd42 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: 049b30eb-57f1-465f-ab2d-b6f8a18762e9 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:01Z" name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd.18a90a1b8e3943f0 namespace: kserve-ci-e2e-test resourceVersion: "13707" uid: 4213d5f0-d9cf-4127-8222-427099100de7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: 049b30eb-57f1-465f-ab2d-b6f8a18762e9 kind: Event lastTimestamp: "2026-04-23T16:44:15Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" in 14.249s (14.249s including waiting). Image size: 1334187097 bytes.' metadata: creationTimestamp: "2026-04-23T16:44:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:15Z" name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd.18a90a1edf96f2ce namespace: kserve-ci-e2e-test resourceVersion: "13880" uid: d786f569-fcaa-44ce-85ff-52198728118e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: 049b30eb-57f1-465f-ab2d-b6f8a18762e9 kind: Event lastTimestamp: "2026-04-23T16:44:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:44:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:15Z" name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd.18a90a1ee418a090 namespace: kserve-ci-e2e-test resourceVersion: "13881" uid: 25184452-4ae9-408d-b3ad-0459d19ba94c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: 049b30eb-57f1-465f-ab2d-b6f8a18762e9 kind: Event lastTimestamp: "2026-04-23T16:44:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:44:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:15Z" name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd.18a90a1ee4935e81 namespace: kserve-ci-e2e-test resourceVersion: "13882" uid: b9e23e22-6742-41ae-98db-70238049e45d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: 049b30eb-57f1-465f-ab2d-b6f8a18762e9 kind: Event lastTimestamp: "2026-04-23T16:44:57Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:57Z" name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd.18a90a1f0d076b56 namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 39ad8b06-f57d-4e65-80c9-70f68cc4b860 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: 049b30eb-57f1-465f-ab2d-b6f8a18762e9 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd.18a90a307b65c25e namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: 4cdefb0b-8287-4034-a4d9-f09c1bc21566 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5bdc4-predictor-66dc784fb6 namespace: kserve-ci-e2e-test resourceVersion: "13649" uid: 5bcf938b-4a49-4bf4-965d-506e4fa7416d kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: 'Created pod: error-404-isvc-5bdc4-predictor-66dc784fb6-4rmbd' metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:01Z" name: error-404-isvc-5bdc4-predictor-66dc784fb6.18a90a1b840ae867 namespace: kserve-ci-e2e-test resourceVersion: "13658" uid: 272dfca7-6cb7-4265-a933-50889c5fc299 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13646" uid: a07139e0-5fd0-4428-81bb-529b0f12b08d kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Scaled up replica set error-404-isvc-5bdc4-predictor-66dc784fb6 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:01Z" name: error-404-isvc-5bdc4-predictor.18a90a1b82a4fac6 namespace: kserve-ci-e2e-test resourceVersion: "13653" uid: 9822f4c1-1ce9-4054-b240-9a0b15058287 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13648" uid: da8658a5-29a1-423e-9b37-ae661eef7d7b kind: Event lastTimestamp: "2026-04-23T16:44:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:31Z" name: error-404-isvc-5bdc4-predictor.18a90a1f013af04a namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: c9fad719-ac76-4c44-b7eb-706a186223c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13648" uid: da8658a5-29a1-423e-9b37-ae661eef7d7b kind: Event lastTimestamp: "2026-04-23T16:44:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:31Z" name: error-404-isvc-5bdc4-predictor.18a90a1f019c8c6f namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: 27ccbc80-b603-41ce-bc22-67d6602591e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:44:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: da8658a5-29a1-423e-9b37-ae661eef7d7b kind: Event lastTimestamp: "2026-04-23T16:45:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:16Z" name: error-404-isvc-5bdc4-predictor.18a90a25ff045f62 namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 53a866ed-5e98-48a9-9d8f-7a11702b1185 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:44:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: da8658a5-29a1-423e-9b37-ae661eef7d7b kind: Event lastTimestamp: "2026-04-23T16:45:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:16Z" name: error-404-isvc-5bdc4-predictor.18a90a25ff64759a namespace: kserve-ci-e2e-test resourceVersion: "14432" uid: b1959d85-31c3-436f-b0af-347e89a3b06f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "13629" uid: 8da1402b-3d7f-4c81-a05d-0276cb5c8c0f kind: Event lastTimestamp: "2026-04-23T16:45:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:07Z" name: error-404-isvc-5bdc4.18a90a1b8271946b namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: 32966b6a-d09a-42f7-b3f3-b07bde384c31 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: 8da1402b-3d7f-4c81-a05d-0276cb5c8c0f kind: Event lastTimestamp: "2026-04-23T16:45:07Z" message: InferenceService [error-404-isvc-5bdc4] is Ready metadata: creationTimestamp: "2026-04-23T16:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:07Z" name: error-404-isvc-5bdc4.18a90a2af0a3973b namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: 38583a28-9ef4-48ec-9e1b-12bf65df22ea reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:55:36.308873Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc namespace: kserve-ci-e2e-test resourceVersion: "19547" uid: 30f124ff-cce0-4699-bc6a-ffc58b185e1f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-65292-predictor-f5b7cdf66-tjrrc to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc.18a90abd5795795a namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: bab84c77-5ff0-4553-aaba-39876815f524 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 30f124ff-cce0-4699-bc6a-ffc58b185e1f kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc.18a90abd5efb78cd namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: cdf59008-d63e-4ebc-8d39-75486380ed34 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 30f124ff-cce0-4699-bc6a-ffc58b185e1f kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc.18a90abd60227a28 namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: 1746c086-0e8a-4a52-9034-d29ec4ea804d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 30f124ff-cce0-4699-bc6a-ffc58b185e1f kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc.18a90abd67d26af3 namespace: kserve-ci-e2e-test resourceVersion: "19573" uid: dffc0dd8-f7d4-4bf0-873f-ddc877916994 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 30f124ff-cce0-4699-bc6a-ffc58b185e1f kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc.18a90abd68446319 namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: b7d60053-9917-48eb-a234-23d4f7bcd5a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 30f124ff-cce0-4699-bc6a-ffc58b185e1f kind: Event lastTimestamp: "2026-04-23T16:56:57Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:57Z" name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc.18a90abd73d1dd96 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 7705960e-acd1-4564-bad4-b9bdef4706f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 30f124ff-cce0-4699-bc6a-ffc58b185e1f kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc.18a90ad002725991 namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: 65514d14-9aa0-423b-9963-48b1cc2fbc33 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-65292-predictor-f5b7cdf66 namespace: kserve-ci-e2e-test resourceVersion: "19538" uid: 66bd39a7-e6de-477d-befa-438b19af5f9e kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: 'Created pod: error-404-isvc-65292-predictor-f5b7cdf66-tjrrc' metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-65292-predictor-f5b7cdf66.18a90abd57331e8a namespace: kserve-ci-e2e-test resourceVersion: "19548" uid: 7e2b881b-d667-4a75-8da5-3097f1aca86a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: 17cb22f1-0515-4b40-ab91-39c530f33ae0 kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Scaled up replica set error-404-isvc-65292-predictor-f5b7cdf66 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-65292-predictor.18a90abd55f5f385 namespace: kserve-ci-e2e-test resourceVersion: "19541" uid: 47e8cdc3-c804-450f-bd16-684ea5063cbc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:55:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: 2c6bad11-401d-4bd8-9d79-cf268c418577 kind: Event lastTimestamp: "2026-04-23T16:56:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:06Z" name: error-404-isvc-65292-predictor.18a90ac0d4b26574 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: c29fb601-9045-48e1-9101-265722f3b8e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:55:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: 2c6bad11-401d-4bd8-9d79-cf268c418577 kind: Event lastTimestamp: "2026-04-23T16:56:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:06Z" name: error-404-isvc-65292-predictor.18a90ac0d50f0984 namespace: kserve-ci-e2e-test resourceVersion: "19847" uid: 6cc44fc8-83c0-4867-bf2b-4bd9283021a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:56:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 2c6bad11-401d-4bd8-9d79-cf268c418577 kind: Event lastTimestamp: "2026-04-23T16:56:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:51Z" name: error-404-isvc-65292-predictor.18a90ac7d31c7d94 namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 660c3400-5343-468b-8b1e-124c87ce52c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:56:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 2c6bad11-401d-4bd8-9d79-cf268c418577 kind: Event lastTimestamp: "2026-04-23T16:56:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:51Z" name: error-404-isvc-65292-predictor.18a90ac7d38bae9a namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: c214097d-7d8a-4f67-a134-672aff508109 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65292 namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: a0d0dbc2-055d-4d38-9141-a568e0459ff6 kind: Event lastTimestamp: "2026-04-23T16:56:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:27Z" name: error-404-isvc-65292.18a90abd55d4aef6 namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: 07ae5540-0dac-4d35-9d2b-35e186bc90ba reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65292 namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: a0d0dbc2-055d-4d38-9141-a568e0459ff6 kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-65292": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-65292": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-65292.18a90abd565aae01 namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: 184ad8bf-f7d4-4642-a97b-a3ba4568e5a8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65292 namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: a0d0dbc2-055d-4d38-9141-a568e0459ff6 kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-65292": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:55:36Z" name: error-404-isvc-65292.18a90abd565b2004 namespace: kserve-ci-e2e-test resourceVersion: "19545" uid: bae0e2af-93cf-4ecd-8c85-dac987efd0a3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65292 namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: a0d0dbc2-055d-4d38-9141-a568e0459ff6 kind: Event lastTimestamp: "2026-04-23T16:56:27Z" message: InferenceService [error-404-isvc-65292] is Ready metadata: creationTimestamp: "2026-04-23T16:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:27Z" name: error-404-isvc-65292.18a90ac957cd4fad namespace: kserve-ci-e2e-test resourceVersion: "20195" uid: 777e81af-eb6f-4eab-85a1-8674c3f012aa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:56:56.542481Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 1e550a89-ceb6-45b4-a611-ec97d98ace2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a7ecd-predictor-5bdddf8766-6g782 to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782.18a90ad005e12c5c namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: d4dc896a-8f62-4c91-ab9c-b616766c8411 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: 1e550a89-ceb6-45b4-a611-ec97d98ace2f kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782.18a90ad00d2b7730 namespace: kserve-ci-e2e-test resourceVersion: "20588" uid: 25b7fa23-f8ec-4aef-858e-7acc73e24137 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782 namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 1e550a89-ceb6-45b4-a611-ec97d98ace2f kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782.18a90ad00e526afc namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 1bf1c8da-ec0d-4772-8b7c-ce38849a8b65 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782 namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 1e550a89-ceb6-45b4-a611-ec97d98ace2f kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782.18a90ad01312ba68 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 6adeeaaf-bd7b-4273-ac93-666ee3ef1022 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782 namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 1e550a89-ceb6-45b4-a611-ec97d98ace2f kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782.18a90ad013889463 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 9230c263-916c-4d02-a640-6ad9b35029ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782 namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 1e550a89-ceb6-45b4-a611-ec97d98ace2f kind: Event lastTimestamp: "2026-04-23T16:57:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:57:38Z" name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782.18a90ad0261158b4 namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 397ea693-2398-4cb6-88b5-a2f2d729c0a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782 namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 1e550a89-ceb6-45b4-a611-ec97d98ace2f kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:21Z" name: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782.18a90b538b2c031c namespace: kserve-ci-e2e-test resourceVersion: "24887" uid: bb61460b-315f-4859-a42d-91dab0f5c773 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a7ecd-predictor-5bdddf8766 namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: 17e407f5-3d97-4ba2-88aa-d8279bd1174b kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: 'Created pod: error-404-isvc-a7ecd-predictor-5bdddf8766-6g782' metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-a7ecd-predictor-5bdddf8766.18a90ad0057ec029 namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 77ee70e3-42d1-4824-98b2-bc108bb26038 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: ee71994d-a567-489d-9ff6-46cf7e9a77de kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Scaled up replica set error-404-isvc-a7ecd-predictor-5bdddf8766 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-a7ecd-predictor.18a90ad0042be3e4 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: 03d3b7cc-1cea-4306-878a-63a52e5d8caa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 699246da-26e5-4754-894a-53b80dc130c1 kind: Event lastTimestamp: "2026-04-23T16:57:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:11Z" name: error-404-isvc-a7ecd-predictor.18a90ad382a1e8bc namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: ea90fca7-dd8c-4e39-9608-478f085cb0df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 699246da-26e5-4754-894a-53b80dc130c1 kind: Event lastTimestamp: "2026-04-23T16:57:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:11Z" name: error-404-isvc-a7ecd-predictor.18a90ad38300246b namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 4ce16bdb-f76e-4a06-bd22-c069663706bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:57:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 699246da-26e5-4754-894a-53b80dc130c1 kind: Event lastTimestamp: "2026-04-23T16:58:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:11Z" name: error-404-isvc-a7ecd-predictor.18a90ad701aa34e1 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 59d7107d-3b48-4cff-bed1-c8cfa92dfa96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:57:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 699246da-26e5-4754-894a-53b80dc130c1 kind: Event lastTimestamp: "2026-04-23T16:58:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:11Z" name: error-404-isvc-a7ecd-predictor.18a90ad7020b9c91 namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: 6adb04af-7d01-4bf8-91e2-abedf92086f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: acc9faaf-8c4e-4b25-860c-c92a9e280245 kind: Event lastTimestamp: "2026-04-23T16:57:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:57:48Z" name: error-404-isvc-a7ecd.18a90ad004079a5a namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: 98aced86-67a6-45e5-af49-611d59824b9e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: acc9faaf-8c4e-4b25-860c-c92a9e280245 kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-a7ecd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a7ecd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-a7ecd.18a90ad0044bf419 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 373f3a13-6402-4918-bbcb-566d18e3f980 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: acc9faaf-8c4e-4b25-860c-c92a9e280245 kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a7ecd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:56Z" name: error-404-isvc-a7ecd.18a90ad0044c3c4d namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: ee8e0f15-7e62-483a-a4ee-2466f29bee4f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21090" uid: acc9faaf-8c4e-4b25-860c-c92a9e280245 kind: Event lastTimestamp: "2026-04-23T16:57:48Z" message: InferenceService [error-404-isvc-a7ecd] is Ready metadata: creationTimestamp: "2026-04-23T16:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:57:48Z" name: error-404-isvc-a7ecd.18a90adc0986821b namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: fa808a90-b553-46cd-bf5f-968c0a97b104 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:45:31.492583Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-be540-predictor-77756f784b-sh7vx namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 09134b41-beba-4d57-91d6-5756496e8ec3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-be540-predictor-77756f784b-sh7vx to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-be540-predictor-77756f784b-sh7vx.18a90a3085b9452a namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: f653a872-6e0c-45af-8ca4-3eebef1c5658 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-be540-predictor-77756f784b-sh7vx namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: 09134b41-beba-4d57-91d6-5756496e8ec3 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-be540-predictor-77756f784b-sh7vx.18a90a308d496afa namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: 3b26b356-523c-4d7a-afee-d9b58fc1b9fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be540-predictor-77756f784b-sh7vx namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 09134b41-beba-4d57-91d6-5756496e8ec3 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-be540-predictor-77756f784b-sh7vx.18a90a308e757a62 namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 671dbb5d-f0ef-47c7-91e7-65cdbb13a00d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be540-predictor-77756f784b-sh7vx namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 09134b41-beba-4d57-91d6-5756496e8ec3 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-be540-predictor-77756f784b-sh7vx.18a90a30956dbc46 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: df4971be-9807-4dc2-95bb-a7a5808cbdba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be540-predictor-77756f784b-sh7vx namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 09134b41-beba-4d57-91d6-5756496e8ec3 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-be540-predictor-77756f784b-sh7vx.18a90a3095df1c1d namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: 7a358e61-3d1b-4556-b557-0a7b2e336df3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be540-predictor-77756f784b-sh7vx namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 09134b41-beba-4d57-91d6-5756496e8ec3 kind: Event lastTimestamp: "2026-04-23T16:46:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:13Z" name: error-404-isvc-be540-predictor-77756f784b-sh7vx.18a90a30cf78504c namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 8c2bbd09-80d1-4bbc-95ef-c2157626da21 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be540-predictor-77756f784b-sh7vx namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 09134b41-beba-4d57-91d6-5756496e8ec3 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-be540-predictor-77756f784b-sh7vx.18a90ab40b91ac06 namespace: kserve-ci-e2e-test resourceVersion: "19094" uid: 8b3ad8cb-b292-40fa-b9b7-b1d32fdce7fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-be540-predictor-77756f784b namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 36b7a4ab-06ca-45f6-ad28-9d6ee915ba2b kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: 'Created pod: error-404-isvc-be540-predictor-77756f784b-sh7vx' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-be540-predictor-77756f784b.18a90a30850173c7 namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: 1abdfbb1-5ba5-4d10-9133-b9da5c0aba00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 8d80a820-e5ce-4e36-9d3c-6e1f23df7abc kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Scaled up replica set error-404-isvc-be540-predictor-77756f784b from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-be540-predictor.18a90a308330dd1c namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: 22c23fe9-23ef-4e6a-8394-43120a161a71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: baf492aa-ed64-4409-9e8b-1c433a863fe8 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: error-404-isvc-be540-predictor.18a90a3401bb1fa6 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 46108424-5365-4742-8e90-07e751e66846 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: baf492aa-ed64-4409-9e8b-1c433a863fe8 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: error-404-isvc-be540-predictor.18a90a34021ba546 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: fa8b79d0-3a28-4809-80d3-17c791cfd7ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: baf492aa-ed64-4409-9e8b-1c433a863fe8 kind: Event lastTimestamp: "2026-04-23T16:46:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:46Z" name: error-404-isvc-be540-predictor.18a90a3affa011cc namespace: kserve-ci-e2e-test resourceVersion: "15604" uid: 454bdd95-5f54-4940-86ce-62159bc403db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: baf492aa-ed64-4409-9e8b-1c433a863fe8 kind: Event lastTimestamp: "2026-04-23T16:46:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:46Z" name: error-404-isvc-be540-predictor.18a90a3b000161a7 namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 69c65c6d-9fe1-4736-be8a-f090234efc5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be540 namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: b8527e0a-c732-46b3-8a4a-5ec552dc28a0 kind: Event lastTimestamp: "2026-04-23T16:46:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:46:23Z" name: error-404-isvc-be540.18a90a3083094cbe namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: 70dd780d-4fe5-4e0c-8282-398b64e8014e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be540 namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: b8527e0a-c732-46b3-8a4a-5ec552dc28a0 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-be540": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-be540": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-be540.18a90a3083681b1f namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 427b357b-bd54-4ce4-bd58-03372f5532cc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be540 namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: b8527e0a-c732-46b3-8a4a-5ec552dc28a0 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-be540": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:31Z" name: error-404-isvc-be540.18a90a3083685bb8 namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: f453cf29-f5d6-4093-ba7e-be51561509de reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be540 namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: b8527e0a-c732-46b3-8a4a-5ec552dc28a0 kind: Event lastTimestamp: "2026-04-23T16:46:23Z" message: InferenceService [error-404-isvc-be540] is Ready metadata: creationTimestamp: "2026-04-23T16:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:46:23Z" name: error-404-isvc-be540.18a90a3cb2cd934d namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: d83557a0-a211-4619-affd-0204bfbf633d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:54:56.441577Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb namespace: kserve-ci-e2e-test resourceVersion: "19118" uid: 3659e25d-c7f7-4979-bfd8-d8ff93d60c90 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb.18a90ab40f4ec9e8 namespace: kserve-ci-e2e-test resourceVersion: "19122" uid: 5a0070c0-cf9d-457e-8db3-57d61dc3a43c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb namespace: kserve-ci-e2e-test resourceVersion: "19128" uid: 3659e25d-c7f7-4979-bfd8-d8ff93d60c90 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb.18a90ab422f23ff9 namespace: kserve-ci-e2e-test resourceVersion: "19141" uid: 00bc165b-8e31-4ccc-ac1f-3e012cc12f92 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb namespace: kserve-ci-e2e-test resourceVersion: "19120" uid: 3659e25d-c7f7-4979-bfd8-d8ff93d60c90 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb.18a90ab424080b62 namespace: kserve-ci-e2e-test resourceVersion: "19143" uid: f13c3e49-da2b-4b55-b2af-3e70b2fee3a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb namespace: kserve-ci-e2e-test resourceVersion: "19120" uid: 3659e25d-c7f7-4979-bfd8-d8ff93d60c90 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb.18a90ab427e63d23 namespace: kserve-ci-e2e-test resourceVersion: "19145" uid: 3b942c1b-a37f-4514-9d49-f5e68246946d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb namespace: kserve-ci-e2e-test resourceVersion: "19120" uid: 3659e25d-c7f7-4979-bfd8-d8ff93d60c90 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb.18a90ab4285ada71 namespace: kserve-ci-e2e-test resourceVersion: "19147" uid: 27079098-a183-444d-9b1a-8a0757c2d641 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T16:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb namespace: kserve-ci-e2e-test resourceVersion: "19120" uid: 3659e25d-c7f7-4979-bfd8-d8ff93d60c90 kind: Event lastTimestamp: "2026-04-23T16:56:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:18Z" name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb.18a90ab456b2ba68 namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: c32f09a2-2a78-4307-bfd3-17aa289ee431 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb namespace: kserve-ci-e2e-test resourceVersion: "19120" uid: 3659e25d-c7f7-4979-bfd8-d8ff93d60c90 kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:16Z" name: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb.18a90ac6bd985210 namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 9043d048-db0d-4bc1-ad2b-8ccfd56fb6d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c88c1-predictor-6df56fdd5b namespace: kserve-ci-e2e-test resourceVersion: "19106" uid: 9d58f6ad-788b-4cd7-a3e4-6710fea8cd10 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: 'Created pod: error-404-isvc-c88c1-predictor-6df56fdd5b-4dnfb' metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-c88c1-predictor-6df56fdd5b.18a90ab40ee37523 namespace: kserve-ci-e2e-test resourceVersion: "19119" uid: 966ffd69-7aa7-45b9-8f1f-3f2a813b3e6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c88c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19103" uid: 5fd27316-d47e-4337-920b-d13aafbdac6b kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Scaled up replica set error-404-isvc-c88c1-predictor-6df56fdd5b from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-c88c1-predictor.18a90ab40d3a7863 namespace: kserve-ci-e2e-test resourceVersion: "19110" uid: aa609af4-43bb-4284-ad09-86717b58d65b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c88c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19104" uid: 796e5244-38dd-4ec3-bfba-6940182508ce kind: Event lastTimestamp: "2026-04-23T16:56:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:11Z" name: error-404-isvc-c88c1-predictor.18a90ab78bbb60da namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: c977f317-b99e-4e7a-9e51-52624e0cb934 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c88c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19104" uid: 796e5244-38dd-4ec3-bfba-6940182508ce kind: Event lastTimestamp: "2026-04-23T16:56:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:11Z" name: error-404-isvc-c88c1-predictor.18a90ab78c17490d namespace: kserve-ci-e2e-test resourceVersion: "19926" uid: 798ba64b-02eb-430b-af25-6b455090abf4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c88c1 namespace: kserve-ci-e2e-test resourceVersion: "19085" uid: a7a467aa-d3b0-435f-b0e5-d4c9b5edf908 kind: Event lastTimestamp: "2026-04-23T16:55:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:55:48Z" name: error-404-isvc-c88c1.18a90ab40d147b83 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 18dd9631-a40c-4737-9e84-14229be30bde reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c88c1 namespace: kserve-ci-e2e-test resourceVersion: "19085" uid: a7a467aa-d3b0-435f-b0e5-d4c9b5edf908 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-c88c1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c88c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-c88c1.18a90ab40d866074 namespace: kserve-ci-e2e-test resourceVersion: "19113" uid: 4d61f221-1afa-48f1-884f-c4a9ca1b9029 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c88c1 namespace: kserve-ci-e2e-test resourceVersion: "19085" uid: a7a467aa-d3b0-435f-b0e5-d4c9b5edf908 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c88c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:54:56Z" name: error-404-isvc-c88c1.18a90ab40d869e07 namespace: kserve-ci-e2e-test resourceVersion: "19116" uid: 853c960e-f0a4-4b08-b281-35833dc58391 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c88c1 namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: a7a467aa-d3b0-435f-b0e5-d4c9b5edf908 kind: Event lastTimestamp: "2026-04-23T16:55:48Z" message: InferenceService [error-404-isvc-c88c1] is Ready metadata: creationTimestamp: "2026-04-23T16:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:55:48Z" name: error-404-isvc-c88c1.18a90ac039fddc74 namespace: kserve-ci-e2e-test resourceVersion: "19702" uid: b77580f8-1716-4223-993b-f1b1aef15916 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:44:01.217647Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13633" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:44:01Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a1b80eb1e0f namespace: kserve-ci-e2e-test resourceVersion: "13640" uid: 9799562e-3d5e-46bd-b3fa-cd7a4a1bf471 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13650" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:44:01Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a1b9a23b783 namespace: kserve-ci-e2e-test resourceVersion: "13730" uid: 2be6e3e6-3844-4153-8038-88f706cb9b44 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:1e3c9913c801ea01a5ba1aef752627a7bb4ea1c0e22f7d100bc3ea27e07d693d" metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:01Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a1b9b609420 namespace: kserve-ci-e2e-test resourceVersion: "13732" uid: f812ee04-9b67-4641-8e2a-6d0f70596cfb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:44:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:1e3c9913c801ea01a5ba1aef752627a7bb4ea1c0e22f7d100bc3ea27e07d693d" in 12.614s (12.614s including waiting). Image size: 299661712 bytes.' metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:14Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a1e8b467f11 namespace: kserve-ci-e2e-test resourceVersion: "13840" uid: a2646564-1882-4a8e-9ee8-372e15b4f65b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:44:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:14Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a1e8ef07ab4 namespace: kserve-ci-e2e-test resourceVersion: "13842" uid: 8e844aa0-e6e6-4f4d-8da5-d9617f985dd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:44:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:14Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a1e8f6e13c0 namespace: kserve-ci-e2e-test resourceVersion: "13844" uid: cc783bb6-8354-41ff-86be-ff27d404278f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:44:18Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1438" metadata: creationTimestamp: "2026-04-23T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:18Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a1f87d36b8d namespace: kserve-ci-e2e-test resourceVersion: "13930" uid: e3473c2d-6750-4575-abcf-bf6ac6f29f59 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:44:25Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1438" in 6.701s (6.701s including waiting). Image size: 1562609571 bytes.' metadata: creationTimestamp: "2026-04-23T16:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:25Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a21173f6686 namespace: kserve-ci-e2e-test resourceVersion: "13992" uid: fa4b1c28-cfde-4902-93e6-3506fd51a5c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:44:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:25Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a211a45f5bf namespace: kserve-ci-e2e-test resourceVersion: "13993" uid: 45864899-343d-4f6f-8b7f-a1d1a327d01f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:44:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:25Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a211ab2998f namespace: kserve-ci-e2e-test resourceVersion: "13996" uid: 1a565a3c-f0fc-4c59-a792-6fe35768b8dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T16:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:45:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:26Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a212a720f40 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: e2dbd3be-fb93-4893-a18a-13731567d866 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: a8d75948-b0cc-4c70-92db-d2d2455d34c9 kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m.18a90a39ce8f498b namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 8717a7f2-5dcb-4a4b-ae25-c3a0a65faefa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-76596dd59f namespace: kserve-ci-e2e-test resourceVersion: "13621" uid: 26091265-b243-49b0-883f-25bc9a5fdc07 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-76596dd59f-mfr6m' metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:01Z" name: isvc-sklearn-graph-1-predictor-76596dd59f.18a90a1b80874a90 namespace: kserve-ci-e2e-test resourceVersion: "13635" uid: 4b372703-996d-456b-988f-7a7482b0a249 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13619" uid: 94975356-a886-4099-b5d3-4b463de062b3 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-76596dd59f from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:01Z" name: isvc-sklearn-graph-1-predictor.18a90a1b7dc4c448 namespace: kserve-ci-e2e-test resourceVersion: "13623" uid: 71785f07-f642-497d-8736-d12f3f242bb6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13620" uid: 3667a0c4-6801-4128-9639-4d55849455b7 kind: Event lastTimestamp: "2026-04-23T16:45:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:01Z" name: isvc-sklearn-graph-1-predictor.18a90a1efc4a6f7e namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: 5e656538-32de-4df7-aea0-5bb31e711cc0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13620" uid: 3667a0c4-6801-4128-9639-4d55849455b7 kind: Event lastTimestamp: "2026-04-23T16:45:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:01Z" name: isvc-sklearn-graph-1-predictor.18a90a1efcb096a8 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 2c069b34-09a5-4681-98e2-e0e35d52ad28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:45:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: 3667a0c4-6801-4128-9639-4d55849455b7 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: isvc-sklearn-graph-1-predictor.18a90a2cf7b472e4 namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: ea706c3e-f1fd-4ba8-ab22-410d86f0a626 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:45:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: 3667a0c4-6801-4128-9639-4d55849455b7 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: isvc-sklearn-graph-1-predictor.18a90a2cf87005f8 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: 7b63c73e-df9c-49e7-b5bf-1b83eea4bd91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13605" uid: 4aeaf042-578c-47ac-923f-8c457a80e6b7 kind: Event lastTimestamp: "2026-04-23T16:45:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:36Z" name: isvc-sklearn-graph-1.18a90a1b7d9b0760 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 5912ead4-c6aa-4753-985d-549204aacc31 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 4aeaf042-578c-47ac-923f-8c457a80e6b7 kind: Event lastTimestamp: "2026-04-23T16:45:36Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-23T16:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:36Z" name: isvc-sklearn-graph-1.18a90a31b88fe1a4 namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: ce056f7d-0d5e-47ac-85cd-0ae20ac7b19f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:44:01.566829Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13717" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-597b747667-ct7xx to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:44:01Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a1b95bb2c72 namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: 34ba941d-ae2c-4de2-8abe-83558c4337e3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13727" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:44:02Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:44:02Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a1bb271f603 namespace: kserve-ci-e2e-test resourceVersion: "13740" uid: fb957e7d-c0d0-48e4-a0c1-c4e79dd7c96d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:44:02Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:1e3c9913c801ea01a5ba1aef752627a7bb4ea1c0e22f7d100bc3ea27e07d693d" metadata: creationTimestamp: "2026-04-23T16:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:02Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a1bb3e0c03d namespace: kserve-ci-e2e-test resourceVersion: "13743" uid: 6c9fbf33-f7df-4657-a625-05c0f2d28823 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:44:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:1e3c9913c801ea01a5ba1aef752627a7bb4ea1c0e22f7d100bc3ea27e07d693d" in 12.203s (12.203s including waiting). Image size: 299661712 bytes.' metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:14Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a1e8b467f6a namespace: kserve-ci-e2e-test resourceVersion: "13841" uid: 6392de5f-6ce1-41ff-8db7-3c74f4d16fbe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:44:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:14Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a1e8efcd7c4 namespace: kserve-ci-e2e-test resourceVersion: "13843" uid: c349be51-dd4a-4b5d-9f62-a0d2624b860d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:44:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:14Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a1e8f7b8d46 namespace: kserve-ci-e2e-test resourceVersion: "13845" uid: 1c19f899-d32e-4660-9b9c-6451e8a126ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:44:18Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1438" metadata: creationTimestamp: "2026-04-23T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:18Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a1f87c198ae namespace: kserve-ci-e2e-test resourceVersion: "13929" uid: 2b382a0f-d30d-42ae-a0a9-49992de239e6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:44:25Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1438" in 6.696s (6.696s including waiting). Image size: 1562609571 bytes.' metadata: creationTimestamp: "2026-04-23T16:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:25Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a2116ecbd8a namespace: kserve-ci-e2e-test resourceVersion: "13990" uid: 30998008-a954-4e72-a268-762781573e79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:44:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:25Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a211a51c08b namespace: kserve-ci-e2e-test resourceVersion: "13995" uid: c86f33dc-5221-41aa-ad58-8236700ed195 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:44:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:25Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a211ac01504 namespace: kserve-ci-e2e-test resourceVersion: "13997" uid: b89a317d-3155-4881-882d-3a1590209905 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T16:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:45:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:26Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a212a65f2c2 namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: a18a5dc8-4e5e-41bd-a608-69b3ac14c9c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b78cea54-3e02-4b9e-9889-6887c5804a7f kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: isvc-sklearn-graph-2-predictor-597b747667-ct7xx.18a90a39cb0e8a9b namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 77870663-881c-4c6e-ad55-541ae8763b8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-597b747667 namespace: kserve-ci-e2e-test resourceVersion: "13697" uid: b7f2c7ea-840b-4e0e-9dfe-2f206a1167ec kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-597b747667-ct7xx' metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:01Z" name: isvc-sklearn-graph-2-predictor-597b747667.18a90a1b95474849 namespace: kserve-ci-e2e-test resourceVersion: "13718" uid: 8f83ce04-c797-4bc0-97b8-8c60bdf6d1d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: 43549cdb-66c8-425e-9887-b53dc8c0b8d1 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-597b747667 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:01Z" name: isvc-sklearn-graph-2-predictor.18a90a1b8c469899 namespace: kserve-ci-e2e-test resourceVersion: "13702" uid: 2fcda31b-92e3-40cb-96ba-04ef7cac3239 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13696" uid: f23d1199-eac6-49a9-b70b-c33790b351ca kind: Event lastTimestamp: "2026-04-23T16:45:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:01Z" name: isvc-sklearn-graph-2-predictor.18a90a1f0ac394eb namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 4399b2d9-83c3-4b5b-9fbe-a7b9c1acdb97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13696" uid: f23d1199-eac6-49a9-b70b-c33790b351ca kind: Event lastTimestamp: "2026-04-23T16:45:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:01Z" name: isvc-sklearn-graph-2-predictor.18a90a1f0b1cb849 namespace: kserve-ci-e2e-test resourceVersion: "14299" uid: 8b893c92-e625-4b6a-b545-b797f199fd66 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:45:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13903" uid: f23d1199-eac6-49a9-b70b-c33790b351ca kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: isvc-sklearn-graph-2-predictor.18a90a2d06ddabcc namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: ceb57271-5ef1-4313-945e-6c5134cba57f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:45:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13903" uid: f23d1199-eac6-49a9-b70b-c33790b351ca kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: isvc-sklearn-graph-2-predictor.18a90a2d07686ccc namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: e20f4d5a-1c89-4a65-9285-140798353456 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13682" uid: ea1b8aab-2bb9-4844-a784-2633849908d1 kind: Event lastTimestamp: "2026-04-23T16:45:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:36Z" name: isvc-sklearn-graph-2.18a90a1b8c1a7e27 namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: 4803d0de-571d-4bee-b10d-2c5749e98e66 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: ea1b8aab-2bb9-4844-a784-2633849908d1 kind: Event lastTimestamp: "2026-04-23T16:45:36Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-23T16:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:36Z" name: isvc-sklearn-graph-2.18a90a31b6b7fda5 namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 2ac6828e-8495-4c15-84e3-c7d6dd996a12 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:44:01.379136Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13683" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-8dp2x to ip-10-0-129-102.ec2.internal metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:44:01Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a1b8a8b3311 namespace: kserve-ci-e2e-test resourceVersion: "13689" uid: 838f5d1d-8d35-42a8-97ab-ff5536b725f5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13698" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:44:01Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a1ba43426cd namespace: kserve-ci-e2e-test resourceVersion: "13733" uid: 9c1f92bb-3e71-4460-94ac-19766ce47465 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:1e3c9913c801ea01a5ba1aef752627a7bb4ea1c0e22f7d100bc3ea27e07d693d" metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:01Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a1ba5b77b69 namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: f229d4e6-7a82-4b94-acd5-4383e0fd9095 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:44:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:1e3c9913c801ea01a5ba1aef752627a7bb4ea1c0e22f7d100bc3ea27e07d693d" in 13.766s (13.766s including waiting). Image size: 299661712 bytes.' metadata: creationTimestamp: "2026-04-23T16:44:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:15Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a1eda4a21a9 namespace: kserve-ci-e2e-test resourceVersion: "13876" uid: 69f41c94-133a-4803-9717-905f19cb4d56 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:44:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T16:44:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:15Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a1ede6541a5 namespace: kserve-ci-e2e-test resourceVersion: "13878" uid: 5ac411cf-c14f-49cf-bba1-116152bb166c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:44:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T16:44:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:15Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a1ededec2e3 namespace: kserve-ci-e2e-test resourceVersion: "13879" uid: 6f634a4a-a47a-408d-95d5-0d1aeabaad59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:44:19Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-23T16:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:19Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a1fc07825c1 namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: d9cee056-ea68-4d60-b445-a9d34afab07b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:44:41Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.628s (21.628s including waiting). Image size: 1306414326 bytes.' metadata: creationTimestamp: "2026-04-23T16:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:41Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a24c99c2df1 namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: d8c5cedb-37e3-478f-8828-fc1357cb4a30 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:44:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:41Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a24d0fbe8f1 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 6e647ff7-6355-42de-81ae-2008361fc9ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:44:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:41Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a24d272fb56 namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: f939e579-542e-4c33-9486-9b21e60fd056 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T16:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:46:12Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:12Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a24e546049c namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 45f18bc8-6a7b-42f5-adf9-fd2f7bdb17b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 822afe44-477a-49a4-99f9-c01638ad964b kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: isvc-xgboost-graph-predictor-784cb989b8-8dp2x.18a90a39d25e7f2b namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: 8f14f644-083e-4529-8c9d-53bbc7dcda09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-102.ec2.internal source: component: kubelet host: ip-10-0-129-102.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 848dcf1b-900e-4d8d-b628-0b25c9a3d784 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-8dp2x' metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:01Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a90a1b8a0744d8 namespace: kserve-ci-e2e-test resourceVersion: "13684" uid: 64543171-b030-4e3a-bf0e-4eb4f31bfb01 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13673" uid: 951eb35e-a5ca-41d8-9385-aec94726d09d kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:01Z" name: isvc-xgboost-graph-predictor.18a90a1b87a9de3f namespace: kserve-ci-e2e-test resourceVersion: "13678" uid: 123826d2-a54d-49c2-b28f-8b3f7a8e4325 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13675" uid: 7e1111ee-c7de-4e61-89a5-1633ecfe4456 kind: Event lastTimestamp: "2026-04-23T16:45:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:16Z" name: isvc-xgboost-graph-predictor.18a90a1f06215408 namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: c38de4ac-ecc6-491e-8f65-4b2ac2a986f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13675" uid: 7e1111ee-c7de-4e61-89a5-1633ecfe4456 kind: Event lastTimestamp: "2026-04-23T16:45:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:16Z" name: isvc-xgboost-graph-predictor.18a90a1f06858ef7 namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: 29af915d-94bc-4c8a-a837-53b44b3c0981 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: 7e1111ee-c7de-4e61-89a5-1633ecfe4456 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: isvc-xgboost-graph-predictor.18a90a3080d6ff1a namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: 89a4244a-5ccc-4b06-9d89-2a5fc2736a26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: 7e1111ee-c7de-4e61-89a5-1633ecfe4456 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: isvc-xgboost-graph-predictor.18a90a3081422666 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 27758bd7-6194-4a0e-aebd-a45376680311 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13661" uid: a0a902e0-c9d7-4f0b-a8a6-081950f27aa6 kind: Event lastTimestamp: "2026-04-23T16:45:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:42Z" name: isvc-xgboost-graph.18a90a1b8781ca3a namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 62fd495f-2d15-4f8a-bb0f-2ccf3cfe1536 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: a0a902e0-c9d7-4f0b-a8a6-081950f27aa6 kind: Event lastTimestamp: "2026-04-23T16:45:42Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-23T16:45:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:42Z" name: isvc-xgboost-graph.18a90a331d792975 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 0bc74b19-0f3b-44aa-969e-bdf26c42d871 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:46:01.117397Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5768cff546-4jftj namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 4c581db6-1683-4dc5-a6dd-b0f330838a9d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5768cff546-4jftj to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:46:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:46:01Z" name: model-chainer-5768cff546-4jftj.18a90a376b8015d4 namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 9e3ec271-2882-498c-b141-5a1f6eb30288 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5768cff546-4jftj namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 4c581db6-1683-4dc5-a6dd-b0f330838a9d kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:46:01Z" name: model-chainer-5768cff546-4jftj.18a90a37844daf9b namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: 4c24a5bd-1fd3-4e42-8d68-2bcbd0aa4fd2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5768cff546-4jftj namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 4c581db6-1683-4dc5-a6dd-b0f330838a9d kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T16:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:01Z" name: model-chainer-5768cff546-4jftj.18a90a3785d73e22 namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 7dc4d64b-9cba-4688-b638-1c8aeb02568c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5768cff546-4jftj namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 4c581db6-1683-4dc5-a6dd-b0f330838a9d kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-23T16:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:01Z" name: model-chainer-5768cff546-4jftj.18a90a37889dec0c namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: c99ed6b6-9880-4ae3-a74d-76cb9de15600 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5768cff546-4jftj namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 4c581db6-1683-4dc5-a6dd-b0f330838a9d kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-23T16:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:01Z" name: model-chainer-5768cff546-4jftj.18a90a37891375c4 namespace: kserve-ci-e2e-test resourceVersion: "15034" uid: ce6b0a54-caba-4026-8b8f-c494ee24d666 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5768cff546-4jftj namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 4c581db6-1683-4dc5-a6dd-b0f330838a9d kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: model-chainer-5768cff546-4jftj.18a90a39c4c0e857 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: 42d681ca-2630-4dd7-962d-5185365b903c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:46:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5768cff546-4jftj namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 4c581db6-1683-4dc5-a6dd-b0f330838a9d kind: Event lastTimestamp: "2026-04-23T16:46:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T16:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:37Z" name: model-chainer-5768cff546-4jftj.18a90a3a249546fa namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 80cf6362-67d6-4286-bb98-25828745ef5b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5768cff546 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: c9aec535-6431-4e66-a486-3ab095873813 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'Created pod: model-chainer-5768cff546-4jftj' metadata: creationTimestamp: "2026-04-23T16:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: model-chainer-5768cff546.18a90a376b169bc1 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 8f2d86ef-ecdd-4765-afc6-d3d30909bfaa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 818cfd23-0151-44fb-87ef-29c139e339b8 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: Scaled up replica set model-chainer-5768cff546 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: model-chainer.18a90a376a60334f namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 9dc614ff-e5fc-48c7-b37a-3f6743335606 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: a5f12ddb-0525-4ea8-84d1-bdb5a709632e kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-23T16:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:46:01Z" name: model-chainer.18a90a376f14b8c7 namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: c3d0b746-a546-478f-914f-75e78e5a2023 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:47:21.376624Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-39400-6c4fd97fc6-7q245 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: f5029734-0ac9-4a1e-8c2a-c27358af7e09 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-39400-6c4fd97fc6-7q245 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:47:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:47:21Z" name: sequence-graph-39400-6c4fd97fc6-7q245.18a90a4a1b52af27 namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 2d6641b5-7359-4288-9c0e-e6a7b4fbab4b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:21Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-39400-6c4fd97fc6-7q245 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: f5029734-0ac9-4a1e-8c2a-c27358af7e09 kind: Event lastTimestamp: "2026-04-23T16:47:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-39400-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T16:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:47:21Z" name: sequence-graph-39400-6c4fd97fc6-7q245.18a90a4a2cb0179f namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 39ff169d-3e5a-4f64-be28-c08ef1907afd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:22Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-39400-6c4fd97fc6-7q245 namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: f5029734-0ac9-4a1e-8c2a-c27358af7e09 kind: Event lastTimestamp: "2026-04-23T16:47:22Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:47:22Z" name: sequence-graph-39400-6c4fd97fc6-7q245.18a90a4a57e09984 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: c21e1c0e-06d2-4766-aa8c-27ba6d91926b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39400} kind: Pod name: sequence-graph-39400-6c4fd97fc6-7q245 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: f5029734-0ac9-4a1e-8c2a-c27358af7e09 kind: Event lastTimestamp: "2026-04-23T16:47:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T16:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:47:22Z" name: sequence-graph-39400-6c4fd97fc6-7q245.18a90a4a5934bdd4 namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: a499da3d-51a2-41fb-ad2f-db915eb41742 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39400} kind: Pod name: sequence-graph-39400-6c4fd97fc6-7q245 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: f5029734-0ac9-4a1e-8c2a-c27358af7e09 kind: Event lastTimestamp: "2026-04-23T16:47:22Z" message: 'Created container: sequence-graph-39400' metadata: creationTimestamp: "2026-04-23T16:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:47:22Z" name: sequence-graph-39400-6c4fd97fc6-7q245.18a90a4a5c0d1081 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: 6a30d823-961d-40e5-890f-d6da605687d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39400} kind: Pod name: sequence-graph-39400-6c4fd97fc6-7q245 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: f5029734-0ac9-4a1e-8c2a-c27358af7e09 kind: Event lastTimestamp: "2026-04-23T16:47:22Z" message: Started container sequence-graph-39400 metadata: creationTimestamp: "2026-04-23T16:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:47:22Z" name: sequence-graph-39400-6c4fd97fc6-7q245.18a90a4a5c787a96 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 42eb1dfe-0ea9-4895-b34c-2a295f4ffc74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39400} kind: Pod name: sequence-graph-39400-6c4fd97fc6-7q245 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: f5029734-0ac9-4a1e-8c2a-c27358af7e09 kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Stopping container sequence-graph-39400 metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:36Z" name: sequence-graph-39400-6c4fd97fc6-7q245.18a90abd4b848ff2 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: 73b618d8-61b7-4b37-9826-771992bc2e91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39400} kind: Pod name: sequence-graph-39400-6c4fd97fc6-7q245 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: f5029734-0ac9-4a1e-8c2a-c27358af7e09 kind: Event lastTimestamp: "2026-04-23T16:56:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:04Z" name: sequence-graph-39400-6c4fd97fc6-7q245.18a90abdfa42f3b4 namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: a4621534-7e16-4588-80d7-4bde3b571e0d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-39400-6c4fd97fc6 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 32475e2f-2ac2-434e-b0c8-b5a87f5f89f4 kind: Event lastTimestamp: "2026-04-23T16:47:21Z" message: 'Created pod: sequence-graph-39400-6c4fd97fc6-7q245' metadata: creationTimestamp: "2026-04-23T16:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:47:21Z" name: sequence-graph-39400-6c4fd97fc6.18a90a4a1ae6ec4e namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: c1768ab1-359e-450e-8e3e-648f4d2f6958 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-39400 namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 6c635792-4e3b-4e68-ba7c-ba0b02dfe98c kind: Event lastTimestamp: "2026-04-23T16:47:21Z" message: Scaled up replica set sequence-graph-39400-6c4fd97fc6 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:47:21Z" name: sequence-graph-39400.18a90a4a1a0ebe2b namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 678796a6-7617-4772-af6c-7c84eb75da6b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-39400 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 24ec6966-8793-4c91-8714-377cfe414e32 kind: Event lastTimestamp: "2026-04-23T16:47:21Z" message: InferenceGraph [sequence-graph-39400] is Ready metadata: creationTimestamp: "2026-04-23T16:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:47:21Z" name: sequence-graph-39400.18a90a4a1e9c4fd9 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 95ba124b-15d5-421b-aa0c-d339bf4ff804 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:47:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-39400 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: f6a31944-a3c5-4f61-a36d-752c217525aa kind: Event lastTimestamp: "2026-04-23T16:48:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:48:06Z" name: sequence-graph-39400.18a90a4d988cba29 namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 1b2b3a47-6b25-441e-bce1-e7542d648383 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:47:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-39400 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: f6a31944-a3c5-4f61-a36d-752c217525aa kind: Event lastTimestamp: "2026-04-23T16:48:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:48:06Z" name: sequence-graph-39400.18a90a4d98ec1707 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 622ceb42-2a1e-4755-868c-fdc8fd1f83d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:56:46.318354Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-65292-5cfbf48c68-6gtrk namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: c29c107e-4f14-49ee-963c-c1e0bffd6fdb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-65292-5cfbf48c68-6gtrk to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:56:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:56:46Z" name: sequence-graph-65292-5cfbf48c68-6gtrk.18a90acda4794f6f namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 7b7cc320-4bbe-40ea-829e-c865046ca29a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-65292-5cfbf48c68-6gtrk namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: c29c107e-4f14-49ee-963c-c1e0bffd6fdb kind: Event lastTimestamp: "2026-04-23T16:56:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-65292-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T16:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:46Z" name: sequence-graph-65292-5cfbf48c68-6gtrk.18a90acdae74a1ed namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 4d2af079-dc9c-4e2a-9b21-4db339705270 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:47Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-65292-5cfbf48c68-6gtrk namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: c29c107e-4f14-49ee-963c-c1e0bffd6fdb kind: Event lastTimestamp: "2026-04-23T16:56:47Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:56:47Z" name: sequence-graph-65292-5cfbf48c68-6gtrk.18a90acde0f6bbab namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: e005603f-0161-4b37-bc1d-5f5014050529 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-65292} kind: Pod name: sequence-graph-65292-5cfbf48c68-6gtrk namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: c29c107e-4f14-49ee-963c-c1e0bffd6fdb kind: Event lastTimestamp: "2026-04-23T16:56:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T16:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:47Z" name: sequence-graph-65292-5cfbf48c68-6gtrk.18a90acde23a6c28 namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 4414e308-aaf5-4afa-9944-ae96a4193ada reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-65292} kind: Pod name: sequence-graph-65292-5cfbf48c68-6gtrk namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: c29c107e-4f14-49ee-963c-c1e0bffd6fdb kind: Event lastTimestamp: "2026-04-23T16:56:47Z" message: 'Created container: sequence-graph-65292' metadata: creationTimestamp: "2026-04-23T16:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:47Z" name: sequence-graph-65292-5cfbf48c68-6gtrk.18a90acde4c049ac namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: feac4bec-8191-4d98-b82b-65487113e438 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-65292} kind: Pod name: sequence-graph-65292-5cfbf48c68-6gtrk namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: c29c107e-4f14-49ee-963c-c1e0bffd6fdb kind: Event lastTimestamp: "2026-04-23T16:56:47Z" message: Started container sequence-graph-65292 metadata: creationTimestamp: "2026-04-23T16:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:47Z" name: sequence-graph-65292-5cfbf48c68-6gtrk.18a90acde52be547 namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: 3354dc45-ac83-40ab-b6c0-bd4c51922804 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-65292} kind: Pod name: sequence-graph-65292-5cfbf48c68-6gtrk namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: c29c107e-4f14-49ee-963c-c1e0bffd6fdb kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Stopping container sequence-graph-65292 metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:56Z" name: sequence-graph-65292-5cfbf48c68-6gtrk.18a90acffa5566b0 namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: e4b68f44-3153-4fe4-861e-eeda9db09f92 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:56:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-65292} kind: Pod name: sequence-graph-65292-5cfbf48c68-6gtrk namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: c29c107e-4f14-49ee-963c-c1e0bffd6fdb kind: Event lastTimestamp: "2026-04-23T16:57:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T16:56:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:57:23Z" name: sequence-graph-65292-5cfbf48c68-6gtrk.18a90ad07be6616b namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: cdffe2de-4791-4d85-8eb1-8770bc4e6ae1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-65292-5cfbf48c68 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 24cf56fa-1662-431b-8ba2-e3306ba0c24c kind: Event lastTimestamp: "2026-04-23T16:56:46Z" message: 'Created pod: sequence-graph-65292-5cfbf48c68-6gtrk' metadata: creationTimestamp: "2026-04-23T16:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:46Z" name: sequence-graph-65292-5cfbf48c68.18a90acda39c774c namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 87c1b244-f912-41ef-a4fe-50110bc35b36 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-65292 namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: 0de01152-adf4-45d3-bacc-61dc1ef676da kind: Event lastTimestamp: "2026-04-23T16:56:46Z" message: Scaled up replica set sequence-graph-65292-5cfbf48c68 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:46Z" name: sequence-graph-65292.18a90acda15d7a99 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2228ed76-7f1d-4859-aed1-cc85428888da reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-65292 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: dd6235c1-2dc6-4cc3-9ebf-5f8d1da157c0 kind: Event lastTimestamp: "2026-04-23T16:56:46Z" message: InferenceGraph [sequence-graph-65292] is Ready metadata: creationTimestamp: "2026-04-23T16:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:46Z" name: sequence-graph-65292.18a90acda81d0fd6 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 348eb440-9e40-4fce-90ec-27da68e6283a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:58:06.557747Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a7ecd-78b4fbf76f-gbcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 13a91d7b-802c-46b5-855c-48425a6dc410 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a7ecd-78b4fbf76f-gbcrh to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:58:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:58:06Z" name: sequence-graph-a7ecd-78b4fbf76f-gbcrh.18a90ae0531d4de1 namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: 01ec1e9d-2fc1-4453-9783-449c85325c94 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:06Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a7ecd-78b4fbf76f-gbcrh namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 13a91d7b-802c-46b5-855c-48425a6dc410 kind: Event lastTimestamp: "2026-04-23T16:58:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-a7ecd-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T16:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:58:06Z" name: sequence-graph-a7ecd-78b4fbf76f-gbcrh.18a90ae06314d7e3 namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 98f6bec5-5f10-4085-920e-3c474722de8d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:07Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a7ecd-78b4fbf76f-gbcrh namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: 13a91d7b-802c-46b5-855c-48425a6dc410 kind: Event lastTimestamp: "2026-04-23T16:58:07Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:58:07Z" name: sequence-graph-a7ecd-78b4fbf76f-gbcrh.18a90ae08f90209d namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: dcf6d4ad-d748-4012-9752-c9e3ccadc3cb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a7ecd} kind: Pod name: sequence-graph-a7ecd-78b4fbf76f-gbcrh namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 13a91d7b-802c-46b5-855c-48425a6dc410 kind: Event lastTimestamp: "2026-04-23T16:58:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T16:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:58:07Z" name: sequence-graph-a7ecd-78b4fbf76f-gbcrh.18a90ae090c3225f namespace: kserve-ci-e2e-test resourceVersion: "21271" uid: e214a6f9-d4ac-48ad-9225-2a980f5f4be3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a7ecd} kind: Pod name: sequence-graph-a7ecd-78b4fbf76f-gbcrh namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 13a91d7b-802c-46b5-855c-48425a6dc410 kind: Event lastTimestamp: "2026-04-23T16:58:07Z" message: 'Created container: sequence-graph-a7ecd' metadata: creationTimestamp: "2026-04-23T16:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:58:07Z" name: sequence-graph-a7ecd-78b4fbf76f-gbcrh.18a90ae0938cd154 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: ebfb2462-7300-4920-b88c-df9b792bb9e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a7ecd} kind: Pod name: sequence-graph-a7ecd-78b4fbf76f-gbcrh namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 13a91d7b-802c-46b5-855c-48425a6dc410 kind: Event lastTimestamp: "2026-04-23T16:58:07Z" message: Started container sequence-graph-a7ecd metadata: creationTimestamp: "2026-04-23T16:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:58:07Z" name: sequence-graph-a7ecd-78b4fbf76f-gbcrh.18a90ae093f9f1a4 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 129bb37c-820e-4aa5-8afa-b0d21b2f83bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a7ecd} kind: Pod name: sequence-graph-a7ecd-78b4fbf76f-gbcrh namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 13a91d7b-802c-46b5-855c-48425a6dc410 kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Stopping container sequence-graph-a7ecd metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:21Z" name: sequence-graph-a7ecd-78b4fbf76f-gbcrh.18a90b537ec593de namespace: kserve-ci-e2e-test resourceVersion: "24816" uid: 7841e422-6e7a-4559-baf2-f44a6619690c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:06:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a7ecd} kind: Pod name: sequence-graph-a7ecd-78b4fbf76f-gbcrh namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 13a91d7b-802c-46b5-855c-48425a6dc410 kind: Event lastTimestamp: "2026-04-23T17:06:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T17:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:48Z" name: sequence-graph-a7ecd-78b4fbf76f-gbcrh.18a90b54160cff1c namespace: kserve-ci-e2e-test resourceVersion: "25161" uid: b3a77204-4f56-47a7-8e09-2a3063d3074a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a7ecd-78b4fbf76f namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: 0a5c3c80-d1d8-44cd-aad8-f85f90421cc8 kind: Event lastTimestamp: "2026-04-23T16:58:06Z" message: 'Created pod: sequence-graph-a7ecd-78b4fbf76f-gbcrh' metadata: creationTimestamp: "2026-04-23T16:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:06Z" name: sequence-graph-a7ecd-78b4fbf76f.18a90ae05294fc70 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: e184363d-71f1-404b-ba27-7567dca3c4b4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: f7856513-8b0d-49de-80c1-bfa8695927d1 kind: Event lastTimestamp: "2026-04-23T16:58:06Z" message: Scaled up replica set sequence-graph-a7ecd-78b4fbf76f from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:06Z" name: sequence-graph-a7ecd.18a90ae051dedb0c namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 102ee8e7-25b5-4ef5-8479-0c7029c76332 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 5bc788a2-1abb-4159-be2b-08eac3367e02 kind: Event lastTimestamp: "2026-04-23T16:58:06Z" message: InferenceGraph [sequence-graph-a7ecd] is Ready metadata: creationTimestamp: "2026-04-23T16:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:58:06Z" name: sequence-graph-a7ecd.18a90ae0569b64e6 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: f3564a2c-f388-492a-aa08-1fbda81b8bea reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 5bc788a2-1abb-4159-be2b-08eac3367e02 kind: Event lastTimestamp: "2026-04-23T16:58:06Z" message: 'Failed to update status for InferenceGraph "sequence-graph-a7ecd": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-a7ecd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:58:06Z" name: sequence-graph-a7ecd.18a90ae058695c30 namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: 1d5fd886-1a07-40c7-9590-76744d27f1bc reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 5bc788a2-1abb-4159-be2b-08eac3367e02 kind: Event lastTimestamp: "2026-04-23T16:58:06Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-a7ecd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:58:06Z" name: sequence-graph-a7ecd.18a90ae05869a4d9 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: fb7189d8-6e23-4678-a531-339c52d3e705 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: df4243f7-c2af-49d2-910e-103c63241316 kind: Event lastTimestamp: "2026-04-23T16:58:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:21Z" name: sequence-graph-a7ecd.18a90ae3d05b1c01 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 6eb7781d-1e28-4696-b6e0-66dfddc9de68 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: df4243f7-c2af-49d2-910e-103c63241316 kind: Event lastTimestamp: "2026-04-23T16:58:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:21Z" name: sequence-graph-a7ecd.18a90ae3d0ba4328 namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: e738a852-fc4e-4297-a1e6-aa6b9ac93dcd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: df4243f7-c2af-49d2-910e-103c63241316 kind: Event lastTimestamp: "2026-04-23T16:58:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:36Z" name: sequence-graph-a7ecd.18a90ae74f6ab31e namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 5dcef9da-2d49-428c-936d-16d432d86eb6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: df4243f7-c2af-49d2-910e-103c63241316 kind: Event lastTimestamp: "2026-04-23T16:58:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:36Z" name: sequence-graph-a7ecd.18a90ae74fcf9e25 namespace: kserve-ci-e2e-test resourceVersion: "21495" uid: ffda3018-d340-4620-bcc8-e7ac04a5352d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T17:06:51.678509Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-56812-687c6bbdcf-mjdzt namespace: kserve-ci-e2e-test resourceVersion: "25199" uid: 1041e52a-41ef-4382-8640-baf093084d81 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-56812-687c6bbdcf-mjdzt to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T17:06:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T17:06:51Z" name: splitter-graph-56812-687c6bbdcf-mjdzt.18a90b5a96c047e4 namespace: kserve-ci-e2e-test resourceVersion: "25204" uid: efaa9001-f22d-4d14-92c3-51bb43e79f99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-56812-687c6bbdcf-mjdzt namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 1041e52a-41ef-4382-8640-baf093084d81 kind: Event lastTimestamp: "2026-04-23T17:06:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-56812-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T17:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:51Z" name: splitter-graph-56812-687c6bbdcf-mjdzt.18a90b5aa186907e namespace: kserve-ci-e2e-test resourceVersion: "25216" uid: ca4fb484-3670-48fb-8338-32e78dc001aa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:52Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-56812-687c6bbdcf-mjdzt namespace: kserve-ci-e2e-test resourceVersion: "25208" uid: 1041e52a-41ef-4382-8640-baf093084d81 kind: Event lastTimestamp: "2026-04-23T17:06:52Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T17:06:52Z" name: splitter-graph-56812-687c6bbdcf-mjdzt.18a90b5ad345dde8 namespace: kserve-ci-e2e-test resourceVersion: "25228" uid: 44a80ef6-9476-46c3-95cb-4ccd72ec5ee3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56812} kind: Pod name: splitter-graph-56812-687c6bbdcf-mjdzt namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 1041e52a-41ef-4382-8640-baf093084d81 kind: Event lastTimestamp: "2026-04-23T17:06:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:52Z" name: splitter-graph-56812-687c6bbdcf-mjdzt.18a90b5ad47d8dbe namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: f3dd6ed5-d9a3-49a3-a53f-bd66fcbb40b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56812} kind: Pod name: splitter-graph-56812-687c6bbdcf-mjdzt namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 1041e52a-41ef-4382-8640-baf093084d81 kind: Event lastTimestamp: "2026-04-23T17:06:52Z" message: 'Created container: splitter-graph-56812' metadata: creationTimestamp: "2026-04-23T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:52Z" name: splitter-graph-56812-687c6bbdcf-mjdzt.18a90b5ad7756a2d namespace: kserve-ci-e2e-test resourceVersion: "25232" uid: f779ade8-16e3-46ef-8b3e-5141a6c85811 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56812} kind: Pod name: splitter-graph-56812-687c6bbdcf-mjdzt namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 1041e52a-41ef-4382-8640-baf093084d81 kind: Event lastTimestamp: "2026-04-23T17:06:52Z" message: Started container splitter-graph-56812 metadata: creationTimestamp: "2026-04-23T17:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:52Z" name: splitter-graph-56812-687c6bbdcf-mjdzt.18a90b5ad7df8da2 namespace: kserve-ci-e2e-test resourceVersion: "25233" uid: 5c7e3429-950e-4d38-8d76-9623bae3fa37 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56812} kind: Pod name: splitter-graph-56812-687c6bbdcf-mjdzt namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 1041e52a-41ef-4382-8640-baf093084d81 kind: Event lastTimestamp: "2026-04-23T17:07:01Z" message: Stopping container splitter-graph-56812 metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:01Z" name: splitter-graph-56812-687c6bbdcf-mjdzt.18a90b5cef4dfd44 namespace: kserve-ci-e2e-test resourceVersion: "25320" uid: a655169e-7a2e-4ebd-96dc-8d4409c2d7bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-56812} kind: Pod name: splitter-graph-56812-687c6bbdcf-mjdzt namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 1041e52a-41ef-4382-8640-baf093084d81 kind: Event lastTimestamp: "2026-04-23T17:07:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T17:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:29Z" name: splitter-graph-56812-687c6bbdcf-mjdzt.18a90b5d7b3426a5 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: a6af082b-f4d4-4066-9f5f-f04e969a8091 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-56812-687c6bbdcf namespace: kserve-ci-e2e-test resourceVersion: "25196" uid: d1d6934d-1de8-408e-91dd-e4cb53576cab kind: Event lastTimestamp: "2026-04-23T17:06:51Z" message: 'Created pod: splitter-graph-56812-687c6bbdcf-mjdzt' metadata: creationTimestamp: "2026-04-23T17:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:51Z" name: splitter-graph-56812-687c6bbdcf.18a90b5a9657ec1b namespace: kserve-ci-e2e-test resourceVersion: "25201" uid: 94ce658e-c590-48a8-b98c-ae5ec3efaf57 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-56812 namespace: kserve-ci-e2e-test resourceVersion: "25194" uid: bd293dba-692d-405b-a274-2a3fa4ea9051 kind: Event lastTimestamp: "2026-04-23T17:06:51Z" message: Scaled up replica set splitter-graph-56812-687c6bbdcf from 0 to 1 metadata: creationTimestamp: "2026-04-23T17:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:51Z" name: splitter-graph-56812.18a90b5a955bc666 namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: 9d257da2-5f7b-4ab2-b60a-4020c40e6ee5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-56812 namespace: kserve-ci-e2e-test resourceVersion: "25210" uid: 05fa3aa8-4792-45dd-8dc0-370dce3b3da4 kind: Event lastTimestamp: "2026-04-23T17:06:51Z" message: InferenceGraph [splitter-graph-56812] is Ready metadata: creationTimestamp: "2026-04-23T17:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:06:51Z" name: splitter-graph-56812.18a90b5a9aade7d5 namespace: kserve-ci-e2e-test resourceVersion: "25212" uid: d72b61b5-391e-4b9d-8ca6-e7f3c730d34d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T17:08:11.960754Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5bc06-756cc58f5d-jkc9m namespace: kserve-ci-e2e-test resourceVersion: "26360" uid: 03838bdb-892d-428b-b289-8fff306cda58 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-5bc06-756cc58f5d-jkc9m to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T17:08:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T17:08:11Z" name: splitter-graph-5bc06-756cc58f5d-jkc9m.18a90b6d47f21a1c namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: c72c1f13-6ebc-40c5-9ac8-3df99395fc18 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:12Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5bc06-756cc58f5d-jkc9m namespace: kserve-ci-e2e-test resourceVersion: "26369" uid: 03838bdb-892d-428b-b289-8fff306cda58 kind: Event lastTimestamp: "2026-04-23T17:08:12Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T17:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T17:08:12Z" name: splitter-graph-5bc06-756cc58f5d-jkc9m.18a90b6d6088f5f1 namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 60dabac1-1baa-447e-8935-472c829057ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5bc06} kind: Pod name: splitter-graph-5bc06-756cc58f5d-jkc9m namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: 03838bdb-892d-428b-b289-8fff306cda58 kind: Event lastTimestamp: "2026-04-23T17:08:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T17:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:08:12Z" name: splitter-graph-5bc06-756cc58f5d-jkc9m.18a90b6d61b4b5c8 namespace: kserve-ci-e2e-test resourceVersion: "26382" uid: 220e21b3-43e4-4820-bfdc-a230f6a0600c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5bc06} kind: Pod name: splitter-graph-5bc06-756cc58f5d-jkc9m namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: 03838bdb-892d-428b-b289-8fff306cda58 kind: Event lastTimestamp: "2026-04-23T17:08:12Z" message: 'Created container: splitter-graph-5bc06' metadata: creationTimestamp: "2026-04-23T17:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:08:12Z" name: splitter-graph-5bc06-756cc58f5d-jkc9m.18a90b6d644db253 namespace: kserve-ci-e2e-test resourceVersion: "26385" uid: 3fb90bcb-5477-4792-bc1c-56bcc0b4f2f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5bc06} kind: Pod name: splitter-graph-5bc06-756cc58f5d-jkc9m namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: 03838bdb-892d-428b-b289-8fff306cda58 kind: Event lastTimestamp: "2026-04-23T17:08:12Z" message: Started container splitter-graph-5bc06 metadata: creationTimestamp: "2026-04-23T17:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:08:12Z" name: splitter-graph-5bc06-756cc58f5d-jkc9m.18a90b6d64b6af93 namespace: kserve-ci-e2e-test resourceVersion: "26386" uid: dce995b9-7818-4490-a214-80f0fc6e9053 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:16:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5bc06} kind: Pod name: splitter-graph-5bc06-756cc58f5d-jkc9m namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: 03838bdb-892d-428b-b289-8fff306cda58 kind: Event lastTimestamp: "2026-04-23T17:16:26Z" message: Stopping container splitter-graph-5bc06 metadata: creationTimestamp: "2026-04-23T17:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:16:26Z" name: splitter-graph-5bc06-756cc58f5d-jkc9m.18a90be07a295f7f namespace: kserve-ci-e2e-test resourceVersion: "29825" uid: 63a4d57e-6425-4ce1-bca6-39c9fcc05be4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5bc06} kind: Pod name: splitter-graph-5bc06-756cc58f5d-jkc9m namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: 03838bdb-892d-428b-b289-8fff306cda58 kind: Event lastTimestamp: "2026-04-23T17:16:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T17:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:16:54Z" name: splitter-graph-5bc06-756cc58f5d-jkc9m.18a90be114bced7e namespace: kserve-ci-e2e-test resourceVersion: "30056" uid: b3f229af-4f11-4e4b-aa8f-19dc1cb20caf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-5bc06-756cc58f5d namespace: kserve-ci-e2e-test resourceVersion: "26357" uid: 6df70a8b-eb88-4b76-8652-6ec37440ab9d kind: Event lastTimestamp: "2026-04-23T17:08:11Z" message: 'Created pod: splitter-graph-5bc06-756cc58f5d-jkc9m' metadata: creationTimestamp: "2026-04-23T17:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:11Z" name: splitter-graph-5bc06-756cc58f5d.18a90b6d478cce52 namespace: kserve-ci-e2e-test resourceVersion: "26361" uid: 1b4cd49f-b7b7-4c59-957b-1e91a3e8d67d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "26355" uid: 9964ed5a-cfb7-48c3-940c-276d66c14aad kind: Event lastTimestamp: "2026-04-23T17:08:11Z" message: Scaled up replica set splitter-graph-5bc06-756cc58f5d from 0 to 1 metadata: creationTimestamp: "2026-04-23T17:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:11Z" name: splitter-graph-5bc06.18a90b6d46dbcbc3 namespace: kserve-ci-e2e-test resourceVersion: "26358" uid: 7ec7bddd-c635-4ac7-830f-f5492ab44b41 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: b61af0cb-76d1-4911-9438-5ea83252d378 kind: Event lastTimestamp: "2026-04-23T17:08:12Z" message: InferenceGraph [splitter-graph-5bc06] is Ready metadata: creationTimestamp: "2026-04-23T17:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:08:12Z" name: splitter-graph-5bc06.18a90b6d4e5bae01 namespace: kserve-ci-e2e-test resourceVersion: "26375" uid: af9105e8-bc23-42bd-b053-8491aa2f0b9e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 3c939a10-fd8e-4137-9adb-21a4284c2be6 kind: Event lastTimestamp: "2026-04-23T17:08:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:26Z" name: splitter-graph-5bc06.18a90b70c579b461 namespace: kserve-ci-e2e-test resourceVersion: "26493" uid: 1568876e-2308-4398-a799-b38011110578 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 3c939a10-fd8e-4137-9adb-21a4284c2be6 kind: Event lastTimestamp: "2026-04-23T17:08:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:26Z" name: splitter-graph-5bc06.18a90b70c5d56f82 namespace: kserve-ci-e2e-test resourceVersion: "26495" uid: f0c7b9c6-f20b-4af3-8cc8-68448edb6dde reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: 3c939a10-fd8e-4137-9adb-21a4284c2be6 kind: Event lastTimestamp: "2026-04-23T17:08:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:41Z" name: splitter-graph-5bc06.18a90b7444d9e76e namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: 84866d11-5818-461d-a119-93bcde1b1a4d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: 3c939a10-fd8e-4137-9adb-21a4284c2be6 kind: Event lastTimestamp: "2026-04-23T17:08:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:41Z" name: splitter-graph-5bc06.18a90b7445386cc9 namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: eecda787-e211-49f6-8f82-2b8afb72e24d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T17:06:21.342253Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: e30c495e-2209-4513-951e-8ca949dd4097 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-390fe-predictor-75d8f85f6f-78jxk to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T17:06:21Z" name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk.18a90b538691bf23 namespace: kserve-ci-e2e-test resourceVersion: "24852" uid: 96a43525-4b28-44de-ae53-5f622bdd9511 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk namespace: kserve-ci-e2e-test resourceVersion: "24857" uid: e30c495e-2209-4513-951e-8ca949dd4097 kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T17:06:21Z" name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk.18a90b538e334507 namespace: kserve-ci-e2e-test resourceVersion: "24901" uid: fd64718e-4261-4509-a068-e1b28229871d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk namespace: kserve-ci-e2e-test resourceVersion: "24850" uid: e30c495e-2209-4513-951e-8ca949dd4097 kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:21Z" name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk.18a90b538f73407e namespace: kserve-ci-e2e-test resourceVersion: "24903" uid: 1ddffde6-d489-49a9-b368-687f6f092304 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk namespace: kserve-ci-e2e-test resourceVersion: "24850" uid: e30c495e-2209-4513-951e-8ca949dd4097 kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:21Z" name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk.18a90b539265529e namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: 398fd04f-5e44-4765-a0b9-4d6f3d80a51c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk namespace: kserve-ci-e2e-test resourceVersion: "24850" uid: e30c495e-2209-4513-951e-8ca949dd4097 kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:21Z" name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk.18a90b5392d48531 namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 2a1fcac8-aaf3-42fc-b979-2b4f2bb0190b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T17:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk namespace: kserve-ci-e2e-test resourceVersion: "24850" uid: e30c495e-2209-4513-951e-8ca949dd4097 kind: Event lastTimestamp: "2026-04-23T17:23:53Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T17:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:23:53Z" name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk.18a90b53aea00d57 namespace: kserve-ci-e2e-test resourceVersion: "32972" uid: 0919d048-806b-436d-afa5-e1a361908908 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk namespace: kserve-ci-e2e-test resourceVersion: "24850" uid: e30c495e-2209-4513-951e-8ca949dd4097 kind: Event lastTimestamp: "2026-04-23T17:23:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:23:50Z" name: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk.18a90c47dd192f06 namespace: kserve-ci-e2e-test resourceVersion: "32913" uid: 8e71fd8f-dc9a-4a80-8f86-d81d295d0a18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-390fe-predictor-75d8f85f6f namespace: kserve-ci-e2e-test resourceVersion: "24830" uid: c84a6e65-5517-4baf-bb89-5191879e6c64 kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: 'Created pod: success-200-isvc-390fe-predictor-75d8f85f6f-78jxk' metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:21Z" name: success-200-isvc-390fe-predictor-75d8f85f6f.18a90b5386265654 namespace: kserve-ci-e2e-test resourceVersion: "24849" uid: f50cc977-7cdf-48c4-9181-67507a580627 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "24827" uid: c91e4fa9-a591-4352-876a-7a5a17e41a6e kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Scaled up replica set success-200-isvc-390fe-predictor-75d8f85f6f from 0 to 1 metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:21Z" name: success-200-isvc-390fe-predictor.18a90b53845fb48b namespace: kserve-ci-e2e-test resourceVersion: "24836" uid: cbd28599-fc99-41b6-8b33-a0f68e4683f5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T17:06:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "24829" uid: 451d47ce-23b5-4d84-b45d-b5dccf68d825 kind: Event lastTimestamp: "2026-04-23T17:07:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:51Z" name: success-200-isvc-390fe-predictor.18a90b57030dcc04 namespace: kserve-ci-e2e-test resourceVersion: "26172" uid: c4725e3e-1b5e-4514-ade6-a63bbd4c4d9a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T17:06:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "24829" uid: 451d47ce-23b5-4d84-b45d-b5dccf68d825 kind: Event lastTimestamp: "2026-04-23T17:07:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:51Z" name: success-200-isvc-390fe-predictor.18a90b5703706108 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: fcd19027-1485-4c66-a5dd-2fae596a321f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "25070" uid: 451d47ce-23b5-4d84-b45d-b5dccf68d825 kind: Event lastTimestamp: "2026-04-23T17:07:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:06Z" name: success-200-isvc-390fe-predictor.18a90b5e00f20d22 namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: 9e77e551-02b7-47f2-85c6-9222d96cecf0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "25070" uid: 451d47ce-23b5-4d84-b45d-b5dccf68d825 kind: Event lastTimestamp: "2026-04-23T17:07:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:06Z" name: success-200-isvc-390fe-predictor.18a90b5e01519e26 namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: 8b568e8b-66fc-47ff-85f1-186278fe49cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390fe namespace: kserve-ci-e2e-test resourceVersion: "24819" uid: 426776c4-add5-4ad8-874a-6df772556de7 kind: Event lastTimestamp: "2026-04-23T17:07:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:13Z" name: success-200-isvc-390fe.18a90b53845b15fa namespace: kserve-ci-e2e-test resourceVersion: "25539" uid: d8d9bd82-16a1-4152-8854-c0b139edd4d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390fe namespace: kserve-ci-e2e-test resourceVersion: "25532" uid: 426776c4-add5-4ad8-874a-6df772556de7 kind: Event lastTimestamp: "2026-04-23T17:07:13Z" message: InferenceService [success-200-isvc-390fe] is Ready metadata: creationTimestamp: "2026-04-23T17:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:13Z" name: success-200-isvc-390fe.18a90b5f92b7cc54 namespace: kserve-ci-e2e-test resourceVersion: "25533" uid: 2aa6366c-814e-4316-b3a0-8cb2449efad2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:46:11.324817Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-39400-predictor-864b86558c-nwrd4 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: b847e371-ddda-4134-9e7c-8cd9ac33a9f5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-39400-predictor-864b86558c-nwrd4 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:46:11Z" name: success-200-isvc-39400-predictor-864b86558c-nwrd4.18a90a39cbe8f5a5 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: b6beb37d-6669-4540-8925-3f3acd23fc47 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-39400-predictor-864b86558c-nwrd4 namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: b847e371-ddda-4134-9e7c-8cd9ac33a9f5 kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:46:11Z" name: success-200-isvc-39400-predictor-864b86558c-nwrd4.18a90a39d3d43cf4 namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: d1bfe411-8f12-467a-90e9-33cb4ea92594 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39400-predictor-864b86558c-nwrd4 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: b847e371-ddda-4134-9e7c-8cd9ac33a9f5 kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: success-200-isvc-39400-predictor-864b86558c-nwrd4.18a90a39d52ce97e namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: 70862d39-9900-471e-a86d-9b7736560cd6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39400-predictor-864b86558c-nwrd4 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: b847e371-ddda-4134-9e7c-8cd9ac33a9f5 kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: success-200-isvc-39400-predictor-864b86558c-nwrd4.18a90a39dbf52f80 namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 12604184-396a-43b0-b288-7325060519ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39400-predictor-864b86558c-nwrd4 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: b847e371-ddda-4134-9e7c-8cd9ac33a9f5 kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:11Z" name: success-200-isvc-39400-predictor-864b86558c-nwrd4.18a90a39dc6cdb63 namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: fec8f273-30df-457d-a8af-88e73f83a227 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39400-predictor-864b86558c-nwrd4 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: b847e371-ddda-4134-9e7c-8cd9ac33a9f5 kind: Event lastTimestamp: "2026-04-23T16:46:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:52Z" name: success-200-isvc-39400-predictor-864b86558c-nwrd4.18a90a39ea3a236f namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: be9372f1-2378-4bb9-8289-69c2f0a2b68c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39400-predictor-864b86558c-nwrd4 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: b847e371-ddda-4134-9e7c-8cd9ac33a9f5 kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-39400-predictor-864b86558c-nwrd4.18a90abd523e442b namespace: kserve-ci-e2e-test resourceVersion: "19507" uid: 5ebe72fd-f064-429f-80e6-076f85f6554b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-39400-predictor-864b86558c namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 23fdc715-e73a-4876-9e41-2edd98dfa4a2 kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: 'Created pod: success-200-isvc-39400-predictor-864b86558c-nwrd4' metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:11Z" name: success-200-isvc-39400-predictor-864b86558c.18a90a39cb886998 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 2a39ab12-4623-4711-902f-02ab9eb0e93c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-39400-predictor namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 148e963a-109d-47d8-832d-5c7a4154aaf3 kind: Event lastTimestamp: "2026-04-23T16:46:11Z" message: Scaled up replica set success-200-isvc-39400-predictor-864b86558c from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:11Z" name: success-200-isvc-39400-predictor.18a90a39ca2171da namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: 16387ff9-8a37-49a7-a8bc-9d44225df445 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:46:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-39400-predictor namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 9ac9cfbd-92a1-4f55-87e3-e7199866c71a kind: Event lastTimestamp: "2026-04-23T16:47:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:47:11Z" name: success-200-isvc-39400-predictor.18a90a3d4884a104 namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: b67a0edb-064c-40a0-bff4-9941644231e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:46:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-39400-predictor namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 9ac9cfbd-92a1-4f55-87e3-e7199866c71a kind: Event lastTimestamp: "2026-04-23T16:47:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:47:11Z" name: success-200-isvc-39400-predictor.18a90a3d48e95eee namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 5e2f4fd5-08f0-4eea-b929-c1275592139b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:46:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-39400 namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 4930a3df-a0b3-4009-ba17-07cb54ca3e48 kind: Event lastTimestamp: "2026-04-23T16:47:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:47:02Z" name: success-200-isvc-39400.18a90a39c9cda2ce namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: f530fc77-dcc5-4baa-8827-ae922b598678 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:47:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-39400 namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: 4930a3df-a0b3-4009-ba17-07cb54ca3e48 kind: Event lastTimestamp: "2026-04-23T16:47:02Z" message: InferenceService [success-200-isvc-39400] is Ready metadata: creationTimestamp: "2026-04-23T16:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:47:02Z" name: success-200-isvc-39400.18a90a45ce2343d9 namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: 75de9487-2e6b-4913-96f1-93f118f03dfc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:56:16.704054Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: a9350d16-cbd5-49a9-a075-8fa3a8cfed3d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:56:16Z" name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r.18a90ac6bf530aad namespace: kserve-ci-e2e-test resourceVersion: "20020" uid: 6497c9c1-f485-417e-895b-0cdce67940e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: a9350d16-cbd5-49a9-a075-8fa3a8cfed3d kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:56:16Z" name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r.18a90ac6c6c994ba namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 16511e47-c75f-4cfd-938c-27c6f51af47e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: a9350d16-cbd5-49a9-a075-8fa3a8cfed3d kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:16Z" name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r.18a90ac6c804c47c namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 0f8c5999-a087-442c-8441-5eee62053f02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: a9350d16-cbd5-49a9-a075-8fa3a8cfed3d kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:16Z" name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r.18a90ac6cad8ff51 namespace: kserve-ci-e2e-test resourceVersion: "20070" uid: 7cc13726-1920-4998-8d78-51ce71131396 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: a9350d16-cbd5-49a9-a075-8fa3a8cfed3d kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:16Z" name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r.18a90ac6cb5e18a5 namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: bebaab45-c6e4-4b34-871a-fc87b0b0b714 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T16:56:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: a9350d16-cbd5-49a9-a075-8fa3a8cfed3d kind: Event lastTimestamp: "2026-04-23T16:56:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:59Z" name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r.18a90ac6eac443cc namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 0ca1cbf9-0eed-4015-9dee-87618981c530 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: a9350d16-cbd5-49a9-a075-8fa3a8cfed3d kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r.18a90b4a42085644 namespace: kserve-ci-e2e-test resourceVersion: "24416" uid: e1bb8a6c-73e8-4016-a180-4fe26f25815b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3ffea-predictor-5bfd58598f namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: 90aac6b9-9f21-49fd-b016-2ff293b31ed6 kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: 'Created pod: success-200-isvc-3ffea-predictor-5bfd58598f-x8f9r' metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:16Z" name: success-200-isvc-3ffea-predictor-5bfd58598f.18a90ac6be8279aa namespace: kserve-ci-e2e-test resourceVersion: "20016" uid: d7309a5a-3626-43e9-a9b2-9a116eb0c091 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: 79ce83af-2f77-4d88-8e89-839de267caa1 kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Scaled up replica set success-200-isvc-3ffea-predictor-5bfd58598f from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:16Z" name: success-200-isvc-3ffea-predictor.18a90ac6bc426993 namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: 7fd7ab07-62b9-4139-91f6-458cfd0eae1e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:56:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: 538c22b4-1876-4d30-b7b4-3cfc92f0a178 kind: Event lastTimestamp: "2026-04-23T16:57:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:31Z" name: success-200-isvc-3ffea-predictor.18a90aca3afb05ee namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: 655fa4be-adf5-40aa-a382-aaff0fcc66fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:56:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: 538c22b4-1876-4d30-b7b4-3cfc92f0a178 kind: Event lastTimestamp: "2026-04-23T16:57:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:31Z" name: success-200-isvc-3ffea-predictor.18a90aca3b6438c1 namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: b38a40bc-0411-49fa-a349-a193d58407c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:57:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: 538c22b4-1876-4d30-b7b4-3cfc92f0a178 kind: Event lastTimestamp: "2026-04-23T16:57:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:16Z" name: success-200-isvc-3ffea-predictor.18a90ad139398361 namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: 3c401051-8fef-418e-b642-669717e4cdb8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:57:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ffea-predictor namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: 538c22b4-1876-4d30-b7b4-3cfc92f0a178 kind: Event lastTimestamp: "2026-04-23T16:57:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:16Z" name: success-200-isvc-3ffea-predictor.18a90ad139a7759d namespace: kserve-ci-e2e-test resourceVersion: "20787" uid: 02e0526f-4428-4a00-88a0-4141c81df783 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ffea namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 6e4d65f5-4df1-492d-8a44-fd34711e35ab kind: Event lastTimestamp: "2026-04-23T16:57:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:57:09Z" name: success-200-isvc-3ffea.18a90ac6bc1d0886 namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 4e202cc9-2e6b-40a1-9f4d-4ce03c02a706 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ffea namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 6e4d65f5-4df1-492d-8a44-fd34711e35ab kind: Event lastTimestamp: "2026-04-23T16:57:09Z" message: InferenceService [success-200-isvc-3ffea] is Ready metadata: creationTimestamp: "2026-04-23T16:57:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:57:09Z" name: success-200-isvc-3ffea.18a90ad30a35b515 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: 34462979-2e8f-40b9-95ee-f69bec34c226 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T17:05:41.601158Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7 namespace: kserve-ci-e2e-test resourceVersion: "24436" uid: 4da01992-da60-469a-8bb9-bd54a37f9e30 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7.18a90b4a45d0c3fb namespace: kserve-ci-e2e-test resourceVersion: "24449" uid: a3a40072-3031-4657-ad26-bcc7aa407c3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7 namespace: kserve-ci-e2e-test resourceVersion: "24460" uid: 4da01992-da60-469a-8bb9-bd54a37f9e30 kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7.18a90b4a4d064726 namespace: kserve-ci-e2e-test resourceVersion: "24475" uid: 94976527-333f-43ed-b66b-b871d547ffd7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7 namespace: kserve-ci-e2e-test resourceVersion: "24441" uid: 4da01992-da60-469a-8bb9-bd54a37f9e30 kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7.18a90b4a4ec0f86b namespace: kserve-ci-e2e-test resourceVersion: "24478" uid: 927da30c-80f4-49f1-9492-9f311d4a3f09 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7 namespace: kserve-ci-e2e-test resourceVersion: "24441" uid: 4da01992-da60-469a-8bb9-bd54a37f9e30 kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7.18a90b4a51b13266 namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: e3c5c255-b513-4a72-8200-e0d194e3865f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7 namespace: kserve-ci-e2e-test resourceVersion: "24441" uid: 4da01992-da60-469a-8bb9-bd54a37f9e30 kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7.18a90b4a521f3cef namespace: kserve-ci-e2e-test resourceVersion: "24488" uid: 768c823b-93a4-4943-8784-85094c79d799 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7 namespace: kserve-ci-e2e-test resourceVersion: "24441" uid: 4da01992-da60-469a-8bb9-bd54a37f9e30 kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:02Z" name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7.18a90b4a57be52b0 namespace: kserve-ci-e2e-test resourceVersion: "25436" uid: 54661858-f06e-4813-a0a3-c5da1652c609 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7 namespace: kserve-ci-e2e-test resourceVersion: "24441" uid: 4da01992-da60-469a-8bb9-bd54a37f9e30 kind: Event lastTimestamp: "2026-04-23T17:07:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:01Z" name: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7.18a90b5cf7f60cc5 namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: a27bdbd6-d617-429c-bb29-14b15ad6450e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-56812-predictor-5dc6bf65f5 namespace: kserve-ci-e2e-test resourceVersion: "24415" uid: 6876048c-3101-4a83-8e8d-32f07adc086a kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: 'Created pod: success-200-isvc-56812-predictor-5dc6bf65f5-tpwn7' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-56812-predictor-5dc6bf65f5.18a90b4a453b6503 namespace: kserve-ci-e2e-test resourceVersion: "24440" uid: 5041ede2-99ad-4e56-bc04-b8d888dfa362 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24411" uid: 35669828-3915-4a77-a381-6325b78173a7 kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: Scaled up replica set success-200-isvc-56812-predictor-5dc6bf65f5 from 0 to 1 metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-56812-predictor.18a90b4a4215b5a7 namespace: kserve-ci-e2e-test resourceVersion: "24417" uid: 36af3c27-2ce1-4c8f-82f3-50d3e2e67473 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:05:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24414" uid: 57d4fa51-58d8-4e04-a996-f445ee421377 kind: Event lastTimestamp: "2026-04-23T17:06:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:11Z" name: success-200-isvc-56812-predictor.18a90b4dc0fb7500 namespace: kserve-ci-e2e-test resourceVersion: "24728" uid: 02895582-aaef-440a-8ad1-0adfd77cb050 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:05:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24414" uid: 57d4fa51-58d8-4e04-a996-f445ee421377 kind: Event lastTimestamp: "2026-04-23T17:06:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:11Z" name: success-200-isvc-56812-predictor.18a90b4dc15aa8d0 namespace: kserve-ci-e2e-test resourceVersion: "24729" uid: 0b51c07e-a2fb-4751-8a86-ebd70a810f0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:06:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24614" uid: 57d4fa51-58d8-4e04-a996-f445ee421377 kind: Event lastTimestamp: "2026-04-23T17:06:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:41Z" name: success-200-isvc-56812-predictor.18a90b54bf23b5f0 namespace: kserve-ci-e2e-test resourceVersion: "25104" uid: d97e1c7f-d473-42c2-b0fd-8fe479404612 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:06:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-56812-predictor namespace: kserve-ci-e2e-test resourceVersion: "24614" uid: 57d4fa51-58d8-4e04-a996-f445ee421377 kind: Event lastTimestamp: "2026-04-23T17:06:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:06:41Z" name: success-200-isvc-56812-predictor.18a90b54bf87228b namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: 35b7e880-5cea-4edd-b9c5-db8bdcade477 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-56812 namespace: kserve-ci-e2e-test resourceVersion: "24395" uid: 00f497f8-8f7d-4c35-a769-21d06a93d3a6 kind: Event lastTimestamp: "2026-04-23T17:06:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:06:33Z" name: success-200-isvc-56812.18a90b4a421c5539 namespace: kserve-ci-e2e-test resourceVersion: "25040" uid: 836f3603-83de-49f5-8f47-1e538a3d1c2e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-56812 namespace: kserve-ci-e2e-test resourceVersion: "24395" uid: 00f497f8-8f7d-4c35-a769-21d06a93d3a6 kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-56812": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-56812": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-56812.18a90b4a4284f497 namespace: kserve-ci-e2e-test resourceVersion: "24420" uid: fc1d5027-1ff0-4303-a94f-f169f1d915be reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-56812 namespace: kserve-ci-e2e-test resourceVersion: "24395" uid: 00f497f8-8f7d-4c35-a769-21d06a93d3a6 kind: Event lastTimestamp: "2026-04-23T17:05:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-56812": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:05:41Z" name: success-200-isvc-56812.18a90b4a428599fa namespace: kserve-ci-e2e-test resourceVersion: "24422" uid: 8933136b-b527-4680-9db8-34d99d794aae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-56812 namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 00f497f8-8f7d-4c35-a769-21d06a93d3a6 kind: Event lastTimestamp: "2026-04-23T17:06:32Z" message: InferenceService [success-200-isvc-56812] is Ready metadata: creationTimestamp: "2026-04-23T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:06:32Z" name: success-200-isvc-56812.18a90b563bffefef namespace: kserve-ci-e2e-test resourceVersion: "25036" uid: d7cfb89c-94d6-4b7b-b33a-9d76ac621c9c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T17:07:01.887794Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc namespace: kserve-ci-e2e-test resourceVersion: "25359" uid: 6780ca19-3b6b-4ac5-98c0-2d25888afd97 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5bc06-predictor-77cbd75659-95hdc to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T17:07:01Z" name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc.18a90b5cf7459c3b namespace: kserve-ci-e2e-test resourceVersion: "25368" uid: 12017c44-62de-461f-bc7a-a334286e771f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: 6780ca19-3b6b-4ac5-98c0-2d25888afd97 kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T17:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T17:07:02Z" name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc.18a90b5cfe38d7f3 namespace: kserve-ci-e2e-test resourceVersion: "25404" uid: 1142a7ba-d05a-40fe-b80f-a7b1a49085c3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 6780ca19-3b6b-4ac5-98c0-2d25888afd97 kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T17:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:02Z" name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc.18a90b5cffe6b5dd namespace: kserve-ci-e2e-test resourceVersion: "25410" uid: 7fc892e8-c3e3-476c-a5b1-e4c54bd7e398 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 6780ca19-3b6b-4ac5-98c0-2d25888afd97 kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T17:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:02Z" name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc.18a90b5d037cf9e1 namespace: kserve-ci-e2e-test resourceVersion: "25412" uid: ec444450-a4e2-4489-a0bd-de4ada76f5af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 6780ca19-3b6b-4ac5-98c0-2d25888afd97 kind: Event lastTimestamp: "2026-04-23T17:07:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T17:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:02Z" name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc.18a90b5d03ed0a1f namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 66c427f8-5ea7-4613-ab3e-fe6aed95eb2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 6780ca19-3b6b-4ac5-98c0-2d25888afd97 kind: Event lastTimestamp: "2026-04-23T17:07:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T17:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:43Z" name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc.18a90b5d04fdbd23 namespace: kserve-ci-e2e-test resourceVersion: "26091" uid: 84e9232c-a911-497e-9be7-fa897a34a1b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:16:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 6780ca19-3b6b-4ac5-98c0-2d25888afd97 kind: Event lastTimestamp: "2026-04-23T17:16:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:16:26Z" name: success-200-isvc-5bc06-predictor-77cbd75659-95hdc.18a90be07f8bbfbd namespace: kserve-ci-e2e-test resourceVersion: "29840" uid: 0ce7d5d6-55d4-408b-89a2-e7dbf26417ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5bc06-predictor-77cbd75659 namespace: kserve-ci-e2e-test resourceVersion: "25347" uid: a58c7446-5b79-48e2-8c75-ac20392f6b95 kind: Event lastTimestamp: "2026-04-23T17:07:01Z" message: 'Created pod: success-200-isvc-5bc06-predictor-77cbd75659-95hdc' metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:01Z" name: success-200-isvc-5bc06-predictor-77cbd75659.18a90b5cf6c06c0c namespace: kserve-ci-e2e-test resourceVersion: "25363" uid: 7aed0ca2-3228-4123-a519-f74f825469b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25344" uid: 5a8ffcc6-403c-45cd-a221-021e14b8660d kind: Event lastTimestamp: "2026-04-23T17:07:01Z" message: Scaled up replica set success-200-isvc-5bc06-predictor-77cbd75659 from 0 to 1 metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:01Z" name: success-200-isvc-5bc06-predictor.18a90b5cf50741ff namespace: kserve-ci-e2e-test resourceVersion: "25353" uid: 79456a5d-c106-4af9-82c9-61de51b694c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:07:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25346" uid: 32d812d1-cdd5-4131-ab84-5b2b67bdc62d kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:31Z" name: success-200-isvc-5bc06-predictor.18a90b6073e00bef namespace: kserve-ci-e2e-test resourceVersion: "25722" uid: 011e0470-36e2-4a03-b4dc-35599e87396c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T17:07:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25346" uid: 32d812d1-cdd5-4131-ab84-5b2b67bdc62d kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:31Z" name: success-200-isvc-5bc06-predictor.18a90b607421a6b4 namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: fca604cb-04be-4e31-b856-5a39dfef115f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T17:07:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: 32d812d1-cdd5-4131-ab84-5b2b67bdc62d kind: Event lastTimestamp: "2026-04-23T17:08:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:16Z" name: success-200-isvc-5bc06-predictor.18a90b6771978218 namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: e2e099d2-c93e-4b72-a2e8-d88c4413e673 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T17:07:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bc06-predictor namespace: kserve-ci-e2e-test resourceVersion: "25573" uid: 32d812d1-cdd5-4131-ab84-5b2b67bdc62d kind: Event lastTimestamp: "2026-04-23T17:08:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:16Z" name: success-200-isvc-5bc06-predictor.18a90b6771f6d379 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 234b2b60-701b-48f8-a839-f0d67737269e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T17:07:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "25326" uid: 1f22892d-26ef-4f5b-a3eb-9de340a76c57 kind: Event lastTimestamp: "2026-04-23T17:07:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T17:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:53Z" name: success-200-isvc-5bc06.18a90b5cf4cd7830 namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: 010cd210-2c40-4d6a-8217-f879ebfb3f34 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bc06 namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: 1f22892d-26ef-4f5b-a3eb-9de340a76c57 kind: Event lastTimestamp: "2026-04-23T17:07:53Z" message: InferenceService [success-200-isvc-5bc06] is Ready metadata: creationTimestamp: "2026-04-23T17:07:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:53Z" name: success-200-isvc-5bc06.18a90b68e8f533b4 namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: 6fb221ed-b6b2-4610-872f-80ecf436cee6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:44:00.970551Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7 namespace: kserve-ci-e2e-test resourceVersion: "13591" uid: c41d37c8-4c2f-41c7-bf52-a20aa8b68262 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:44:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:44:00Z" name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7.18a90a1b7230a696 namespace: kserve-ci-e2e-test resourceVersion: "13596" uid: 1d120206-ab76-47e6-8754-222dc3dfb910 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7 namespace: kserve-ci-e2e-test resourceVersion: "13601" uid: c41d37c8-4c2f-41c7-bf52-a20aa8b68262 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Add eth0 [10.132.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:44:01Z" name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7.18a90a1b79187320 namespace: kserve-ci-e2e-test resourceVersion: "13608" uid: 6d00ba0c-a5ce-4322-9268-23b3649d5fa6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7 namespace: kserve-ci-e2e-test resourceVersion: "13593" uid: c41d37c8-4c2f-41c7-bf52-a20aa8b68262 kind: Event lastTimestamp: "2026-04-23T16:44:01Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:01Z" name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7.18a90a1b7a57d45c namespace: kserve-ci-e2e-test resourceVersion: "13612" uid: 2a4eee31-0a90-45cb-a38d-68a5a2c3be90 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7 namespace: kserve-ci-e2e-test resourceVersion: "13593" uid: c41d37c8-4c2f-41c7-bf52-a20aa8b68262 kind: Event lastTimestamp: "2026-04-23T16:44:14Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" in 13.257s (13.257s including waiting). Image size: 1334246496 bytes.' metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:14Z" name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7.18a90a1e9093efd9 namespace: kserve-ci-e2e-test resourceVersion: "13846" uid: d3155d18-6106-4eee-b83a-d88b5977a71c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7 namespace: kserve-ci-e2e-test resourceVersion: "13593" uid: c41d37c8-4c2f-41c7-bf52-a20aa8b68262 kind: Event lastTimestamp: "2026-04-23T16:44:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:14Z" name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7.18a90a1e94338803 namespace: kserve-ci-e2e-test resourceVersion: "13849" uid: fa4ebb69-a94f-4004-a423-7c872f32f82a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7 namespace: kserve-ci-e2e-test resourceVersion: "13593" uid: c41d37c8-4c2f-41c7-bf52-a20aa8b68262 kind: Event lastTimestamp: "2026-04-23T16:44:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:14Z" name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7.18a90a1e94a8a820 namespace: kserve-ci-e2e-test resourceVersion: "13850" uid: 83281f07-b304-4103-8086-eb459525a85c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:44:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7 namespace: kserve-ci-e2e-test resourceVersion: "13593" uid: c41d37c8-4c2f-41c7-bf52-a20aa8b68262 kind: Event lastTimestamp: "2026-04-23T16:44:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:44:55Z" name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7.18a90a1e989655db namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: ff1c7ad1-1839-477a-8da7-a4de2c1d6636 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7 namespace: kserve-ci-e2e-test resourceVersion: "13593" uid: c41d37c8-4c2f-41c7-bf52-a20aa8b68262 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7.18a90a307e9940e7 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: c8e17d84-0c8c-4c92-b28e-149e52dacc95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5bdc4-predictor-d6748fc7f namespace: kserve-ci-e2e-test resourceVersion: "13582" uid: 8eb1d6ff-ec89-4ecf-ad4a-696cf043bdd2 kind: Event lastTimestamp: "2026-04-23T16:44:00Z" message: 'Created pod: success-200-isvc-5bdc4-predictor-d6748fc7f-6x4x7' metadata: creationTimestamp: "2026-04-23T16:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:00Z" name: success-200-isvc-5bdc4-predictor-d6748fc7f.18a90a1b71a9a124 namespace: kserve-ci-e2e-test resourceVersion: "13594" uid: b4acf2d5-74cc-403b-aed3-f97644b48e4c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13580" uid: 17d3d687-1e7d-43d6-b763-6eaca844492d kind: Event lastTimestamp: "2026-04-23T16:44:00Z" message: Scaled up replica set success-200-isvc-5bdc4-predictor-d6748fc7f from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:00Z" name: success-200-isvc-5bdc4-predictor.18a90a1b6cd05f05 namespace: kserve-ci-e2e-test resourceVersion: "13584" uid: 46181e14-fdc5-4142-ab71-3a54dc9c00f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 54b45e3f-0a4a-4f5d-af77-5765f2948031 kind: Event lastTimestamp: "2026-04-23T16:44:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:46Z" name: success-200-isvc-5bdc4-predictor.18a90a1ef275c6cb namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 623f686e-d4bc-4cec-b131-0b761844ca75 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 54b45e3f-0a4a-4f5d-af77-5765f2948031 kind: Event lastTimestamp: "2026-04-23T16:44:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:44:46Z" name: success-200-isvc-5bdc4-predictor.18a90a1ef2d69a77 namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 329550cd-c5eb-4c65-843b-83ba38078663 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:45:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13887" uid: 54b45e3f-0a4a-4f5d-af77-5765f2948031 kind: Event lastTimestamp: "2026-04-23T16:45:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:16Z" name: success-200-isvc-5bdc4-predictor.18a90a296f6e6ba6 namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: ba09911b-2f4b-4058-baa0-34071dfd4d4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:45:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bdc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13887" uid: 54b45e3f-0a4a-4f5d-af77-5765f2948031 kind: Event lastTimestamp: "2026-04-23T16:45:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:16Z" name: success-200-isvc-5bdc4-predictor.18a90a296fd71dcc namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: 0f47e8c9-92a5-4f2f-acd5-4185028ba28c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:44:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: ef6327a5-3369-4c6f-91d7-0193f1e9fdd8 kind: Event lastTimestamp: "2026-04-23T16:45:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:05Z" name: success-200-isvc-5bdc4.18a90a1b731d69ae namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: 43d56631-9511-45bb-8134-d3d0e7680a19 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: ef6327a5-3369-4c6f-91d7-0193f1e9fdd8 kind: Event lastTimestamp: "2026-04-23T16:44:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-5bdc4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5bdc4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:44:00Z" name: success-200-isvc-5bdc4.18a90a1b73ba7299 namespace: kserve-ci-e2e-test resourceVersion: "13603" uid: f7ad536c-1c48-444c-8c01-8fa8804466a0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:44:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "13570" uid: ef6327a5-3369-4c6f-91d7-0193f1e9fdd8 kind: Event lastTimestamp: "2026-04-23T16:44:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5bdc4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:44:01Z" name: success-200-isvc-5bdc4.18a90a1b73badc25 namespace: kserve-ci-e2e-test resourceVersion: "13604" uid: c08e7077-3bb9-486f-b498-4ab398dc46e0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: ef6327a5-3369-4c6f-91d7-0193f1e9fdd8 kind: Event lastTimestamp: "2026-04-23T16:45:05Z" message: InferenceService [success-200-isvc-5bdc4] is Ready metadata: creationTimestamp: "2026-04-23T16:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:05Z" name: success-200-isvc-5bdc4.18a90a2a7ca05b02 namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: 105dc3f3-5794-4bb2-a1aa-5c498c3a0f73 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:55:36.240201Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: 99de4ace-3a2d-4fee-8d6f-5e6f79b6e79f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-65292-predictor-67b8bb4fcf-clsjg to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg.18a90abd537da05e namespace: kserve-ci-e2e-test resourceVersion: "19516" uid: 0fbc7900-2e7e-4baa-a599-b94755d50641 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg namespace: kserve-ci-e2e-test resourceVersion: "19523" uid: 99de4ace-3a2d-4fee-8d6f-5e6f79b6e79f kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg.18a90abd5b57bdc0 namespace: kserve-ci-e2e-test resourceVersion: "19560" uid: 7f88d14d-adc1-42ed-8030-0a885e747243 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg namespace: kserve-ci-e2e-test resourceVersion: "19512" uid: 99de4ace-3a2d-4fee-8d6f-5e6f79b6e79f kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg.18a90abd5c9b3549 namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: b854e13a-26a5-412d-a330-d0615b93d868 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg namespace: kserve-ci-e2e-test resourceVersion: "19512" uid: 99de4ace-3a2d-4fee-8d6f-5e6f79b6e79f kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg.18a90abd6290d49a namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: 2ffa508b-6187-404c-8ee6-4393d68dbb13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg namespace: kserve-ci-e2e-test resourceVersion: "19512" uid: 99de4ace-3a2d-4fee-8d6f-5e6f79b6e79f kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg.18a90abd63053d3c namespace: kserve-ci-e2e-test resourceVersion: "19571" uid: 03923dfe-3b73-4afc-8411-8ecdba0092db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T16:55:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg namespace: kserve-ci-e2e-test resourceVersion: "19512" uid: 99de4ace-3a2d-4fee-8d6f-5e6f79b6e79f kind: Event lastTimestamp: "2026-04-23T16:56:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:55:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:58Z" name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg.18a90abd937a9804 namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7eb8ab88-c5ed-403b-9894-24af5b0ade3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg namespace: kserve-ci-e2e-test resourceVersion: "19512" uid: 99de4ace-3a2d-4fee-8d6f-5e6f79b6e79f kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:56Z" name: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg.18a90acfff73096c namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 20aac853-5ab4-4ec2-9229-8ad1f51be1c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-65292-predictor-67b8bb4fcf namespace: kserve-ci-e2e-test resourceVersion: "19494" uid: 3b7d0fc7-2e42-4f5e-b56e-faf2f9142795 kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: 'Created pod: success-200-isvc-65292-predictor-67b8bb4fcf-clsjg' metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-65292-predictor-67b8bb4fcf.18a90abd52f8feba namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: 4a64d0d0-f98f-41ba-b27d-b0dc061f956d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19490" uid: 5b8c83cb-9629-4bb1-abe9-4d2704145a92 kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: Scaled up replica set success-200-isvc-65292-predictor-67b8bb4fcf from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-65292-predictor.18a90abd5106b0e3 namespace: kserve-ci-e2e-test resourceVersion: "19500" uid: cffd321c-a557-4f7b-bb6b-f92749261863 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: 117e2386-a193-4b58-9633-1840a0fed202 kind: Event lastTimestamp: "2026-04-23T16:55:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:55:51Z" name: success-200-isvc-65292-predictor.18a90ac0cfc4fc4e namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: efe5ce61-0fe7-42bf-bbe4-e1904d774af5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: 117e2386-a193-4b58-9633-1840a0fed202 kind: Event lastTimestamp: "2026-04-23T16:55:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:55:51Z" name: success-200-isvc-65292-predictor.18a90ac0d0277564 namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 5d804b5f-c2c8-4116-9295-d99d929f883d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:56:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 117e2386-a193-4b58-9633-1840a0fed202 kind: Event lastTimestamp: "2026-04-23T16:56:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:51Z" name: success-200-isvc-65292-predictor.18a90ac44ec608d7 namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: bb300a27-dead-49cf-9e76-4e1d2f7cea10 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T16:56:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65292-predictor namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 117e2386-a193-4b58-9633-1840a0fed202 kind: Event lastTimestamp: "2026-04-23T16:56:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:51Z" name: success-200-isvc-65292-predictor.18a90ac44f257856 namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: 11dfad88-0e85-49fd-a61d-67bdfb570b03 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65292 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 986ab5d6-15e5-4200-8a10-2a43dc5c9869 kind: Event lastTimestamp: "2026-04-23T16:56:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:28Z" name: success-200-isvc-65292.18a90abd50e198ae namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: b6212017-9b88-4f10-a4e6-346468bece92 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65292 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 986ab5d6-15e5-4200-8a10-2a43dc5c9869 kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-65292": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-65292": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-65292.18a90abd517b9595 namespace: kserve-ci-e2e-test resourceVersion: "19502" uid: f0acad4a-dc3d-4b01-9c8b-0b87637f94a0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65292 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 986ab5d6-15e5-4200-8a10-2a43dc5c9869 kind: Event lastTimestamp: "2026-04-23T16:55:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-65292": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:55:36Z" name: success-200-isvc-65292.18a90abd517be4a4 namespace: kserve-ci-e2e-test resourceVersion: "19505" uid: 34cf8521-a445-4bee-9a36-22c5e3cb6159 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65292 namespace: kserve-ci-e2e-test resourceVersion: "20207" uid: 986ab5d6-15e5-4200-8a10-2a43dc5c9869 kind: Event lastTimestamp: "2026-04-23T16:56:28Z" message: InferenceService [success-200-isvc-65292] is Ready metadata: creationTimestamp: "2026-04-23T16:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:56:28Z" name: success-200-isvc-65292.18a90ac9774db68b namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: 0e55dddf-ba6f-4ba4-b858-8994fe52faee reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:56:56.469440Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: d948d245-4a2d-49b9-a873-32c041bc5cc0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a7ecd-predictor-76b497db67-9wwds to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:56:56Z" name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds.18a90ad00186993d namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: d7a05cf1-56db-4623-9f2e-d67e95d521ef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: d948d245-4a2d-49b9-a873-32c041bc5cc0 kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:56:56Z" name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds.18a90ad008f652b8 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 4c8bc7b9-8fdc-4663-a8ae-1b213da41f7a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: d948d245-4a2d-49b9-a873-32c041bc5cc0 kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:56Z" name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds.18a90ad00a12943b namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: 12a40935-bb45-4116-9e9a-6fd3c7a97c9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: d948d245-4a2d-49b9-a873-32c041bc5cc0 kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:56Z" name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds.18a90ad00d2729ca namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: 4c349b65-bc1f-4fa6-a396-502fb867ff63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: d948d245-4a2d-49b9-a873-32c041bc5cc0 kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:56Z" name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds.18a90ad00d92806a namespace: kserve-ci-e2e-test resourceVersion: "20590" uid: f1959a43-706a-471e-afef-352e14ac1881 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: d948d245-4a2d-49b9-a873-32c041bc5cc0 kind: Event lastTimestamp: "2026-04-23T16:57:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:57:38Z" name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds.18a90ad041c5b85a namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: b58ca7c2-f3b3-4a50-8cb9-b502b4d3e969 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: d948d245-4a2d-49b9-a873-32c041bc5cc0 kind: Event lastTimestamp: "2026-04-23T17:06:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:06:21Z" name: success-200-isvc-a7ecd-predictor-76b497db67-9wwds.18a90b5385eb34d3 namespace: kserve-ci-e2e-test resourceVersion: "24847" uid: 04a8a019-a9c5-48ad-babd-306b77ae3d11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a7ecd-predictor-76b497db67 namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: d85b13fc-b1c4-468e-9969-2e92cbc92fd2 kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: 'Created pod: success-200-isvc-a7ecd-predictor-76b497db67-9wwds' metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:56Z" name: success-200-isvc-a7ecd-predictor-76b497db67.18a90ad0010c7a17 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: da7cabb0-819c-4c5d-bef1-d4d9be146e3e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: d726a49a-8063-4a1d-9ac0-b22397317d51 kind: Event lastTimestamp: "2026-04-23T16:56:56Z" message: Scaled up replica set success-200-isvc-a7ecd-predictor-76b497db67 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:56Z" name: success-200-isvc-a7ecd-predictor.18a90acfffa6ce69 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 31cc852e-e76c-46f1-b15d-476ad9797a85 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:57:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 639f02da-02f9-45d3-b84e-2673eb3ec9fa kind: Event lastTimestamp: "2026-04-23T16:57:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:26Z" name: success-200-isvc-a7ecd-predictor.18a90ad37e6da674 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 278db79a-4c59-40e5-86b4-b634edddc1ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:57:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 639f02da-02f9-45d3-b84e-2673eb3ec9fa kind: Event lastTimestamp: "2026-04-23T16:57:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:57:26Z" name: success-200-isvc-a7ecd-predictor.18a90ad37ecf0320 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: b9a3e456-8ba5-4580-8145-952489103d5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:57:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 639f02da-02f9-45d3-b84e-2673eb3ec9fa kind: Event lastTimestamp: "2026-04-23T16:58:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:11Z" name: success-200-isvc-a7ecd-predictor.18a90ada7c80dabf namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 3042aa42-346e-4de6-87d1-02976476aa36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:57:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a7ecd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 639f02da-02f9-45d3-b84e-2673eb3ec9fa kind: Event lastTimestamp: "2026-04-23T16:58:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:58:11Z" name: success-200-isvc-a7ecd-predictor.18a90ada7cdbd181 namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: 781f45c6-76ff-4f88-b0f4-c5c4fa24dcad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:56:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "20501" uid: 4bb3c91e-de95-42b6-8442-11a379c7a18f kind: Event lastTimestamp: "2026-04-23T16:57:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:57:48Z" name: success-200-isvc-a7ecd.18a90acfff7ee413 namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: bc1d4716-b58d-417b-aa8c-b24305121e4d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:57:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a7ecd namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 4bb3c91e-de95-42b6-8442-11a379c7a18f kind: Event lastTimestamp: "2026-04-23T16:57:48Z" message: InferenceService [success-200-isvc-a7ecd] is Ready metadata: creationTimestamp: "2026-04-23T16:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:57:48Z" name: success-200-isvc-a7ecd.18a90adc256ba27a namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: 739d2797-38b5-4274-a473-5b4ea3a21456 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:45:31.399193Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: f0728115-e6b3-4748-9502-0f79c2fba7b0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-be540-predictor-5b5665dc9-8cfm7 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7.18a90a308028466a namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: 1d1147f1-d178-4dd5-9c30-188dc8c5e4b2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7 namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: f0728115-e6b3-4748-9502-0f79c2fba7b0 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7.18a90a3087124ece namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 9291351f-c5e3-49db-a463-81de5600a01f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7 namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: f0728115-e6b3-4748-9502-0f79c2fba7b0 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7.18a90a30887b5f74 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 8c570e13-1692-4fb2-a0e2-7d66c66a8ff6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7 namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: f0728115-e6b3-4748-9502-0f79c2fba7b0 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7.18a90a308bb5472e namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: b9902196-d799-474e-83bc-3fca8cc57bce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7 namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: f0728115-e6b3-4748-9502-0f79c2fba7b0 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7.18a90a308c275c4d namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 1261cf47-6b13-4fbd-b9c2-d8c10e64a2bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7 namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: f0728115-e6b3-4748-9502-0f79c2fba7b0 kind: Event lastTimestamp: "2026-04-23T16:46:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:12Z" name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7.18a90a3093460d48 namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 10659cea-a5c7-4d00-873f-a96340bd3e39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7 namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: f0728115-e6b3-4748-9502-0f79c2fba7b0 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:54:56Z" name: success-200-isvc-be540-predictor-5b5665dc9-8cfm7.18a90ab408f30066 namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: ecb06e41-4dbd-4650-b6e9-f1f704b9d354 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-be540-predictor-5b5665dc9 namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: 1e877ee4-18b1-4a29-931f-09ad5de37513 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: 'Created pod: success-200-isvc-be540-predictor-5b5665dc9-8cfm7' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-be540-predictor-5b5665dc9.18a90a307fa17529 namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 168f8bf0-e12c-4421-826a-d37963569d38 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: e87d37c6-a285-4241-8578-8244fded4b7b kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Scaled up replica set success-200-isvc-be540-predictor-5b5665dc9 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-be540-predictor.18a90a307dce77c2 namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: b98e86b8-0b07-403f-802d-9f324890885e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: e786a196-0f87-4233-969e-b31c64de6b58 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: success-200-isvc-be540-predictor.18a90a33fc95df91 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 5c0e0532-1365-4ebf-ad43-276abf5b306f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: e786a196-0f87-4233-969e-b31c64de6b58 kind: Event lastTimestamp: "2026-04-23T16:46:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:01Z" name: success-200-isvc-be540-predictor.18a90a33fcfd6f02 namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: f024e828-fe38-42a9-b74b-c1522a8ba4a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: e786a196-0f87-4233-969e-b31c64de6b58 kind: Event lastTimestamp: "2026-04-23T16:46:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:46Z" name: success-200-isvc-be540-predictor.18a90a3afa7c3c99 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 89ae03ec-937b-4a03-92a5-b8dbde464c61 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be540-predictor namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: e786a196-0f87-4233-969e-b31c64de6b58 kind: Event lastTimestamp: "2026-04-23T16:46:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:46Z" name: success-200-isvc-be540-predictor.18a90a3afae73ebf namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: e213df9d-2276-4335-8a60-a38c967b5e44 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be540 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: fdfd55bc-0eb9-4c90-9402-8f01023a5020 kind: Event lastTimestamp: "2026-04-23T16:46:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:46:22Z" name: success-200-isvc-be540.18a90a307df6844a namespace: kserve-ci-e2e-test resourceVersion: "15360" uid: 9e178258-9ba7-4c07-b36b-fdc8257489f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be540 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: fdfd55bc-0eb9-4c90-9402-8f01023a5020 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-be540": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-be540": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-be540.18a90a307e7d7f70 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 971a6e8a-eeac-4c3f-8b0a-4da4a985a428 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be540 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: fdfd55bc-0eb9-4c90-9402-8f01023a5020 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-be540": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:31Z" name: success-200-isvc-be540.18a90a307e7dbbd5 namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 3987c699-d99c-4b12-97f3-9ce4eaec6702 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be540 namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: fdfd55bc-0eb9-4c90-9402-8f01023a5020 kind: Event lastTimestamp: "2026-04-23T16:46:22Z" message: InferenceService [success-200-isvc-be540] is Ready metadata: creationTimestamp: "2026-04-23T16:46:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:46:22Z" name: success-200-isvc-be540.18a90a3c76be84f3 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: e4b4046a-f315-420f-b2ff-33fc698a08ad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:54:56.396751Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6 namespace: kserve-ci-e2e-test resourceVersion: "19097" uid: a7fe4523-9d21-4b07-9037-141b73c6c9ea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c88c1-predictor-5c979449cc-hqsj6 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:54:56Z" name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6.18a90ab40ca2cf20 namespace: kserve-ci-e2e-test resourceVersion: "19107" uid: 00edb706-3e1f-4865-8b94-34f41b53c4af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6 namespace: kserve-ci-e2e-test resourceVersion: "19115" uid: a7fe4523-9d21-4b07-9037-141b73c6c9ea kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:54:56Z" name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6.18a90ab413ef510a namespace: kserve-ci-e2e-test resourceVersion: "19133" uid: 715dabd5-20a6-4d46-adda-9b3b9270f3bb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: a7fe4523-9d21-4b07-9037-141b73c6c9ea kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1438" already present on machine metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:54:56Z" name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6.18a90ab4152e3939 namespace: kserve-ci-e2e-test resourceVersion: "19136" uid: 075aa110-69e5-49ce-a502-b9b233023f10 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: a7fe4523-9d21-4b07-9037-141b73c6c9ea kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:54:56Z" name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6.18a90ab417ba3f2f namespace: kserve-ci-e2e-test resourceVersion: "19137" uid: 032d6c44-6355-45a6-aee2-b770c42c1316 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: a7fe4523-9d21-4b07-9037-141b73c6c9ea kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:54:56Z" name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6.18a90ab41824859d namespace: kserve-ci-e2e-test resourceVersion: "19138" uid: 5006efa4-bf5c-4a55-a52f-00ff008450ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-23T16:54:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: a7fe4523-9d21-4b07-9037-141b73c6c9ea kind: Event lastTimestamp: "2026-04-23T16:56:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T16:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:18Z" name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6.18a90ab43caa175f namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 963172e9-695d-40e0-bbf3-b411be0a8017 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: a7fe4523-9d21-4b07-9037-141b73c6c9ea kind: Event lastTimestamp: "2026-04-23T16:56:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T16:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:56:16Z" name: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6.18a90ac6c201f845 namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 1be05223-e1ea-4ebe-862e-a53336898af2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c88c1-predictor-5c979449cc namespace: kserve-ci-e2e-test resourceVersion: "19072" uid: cb8ac9eb-fe57-43db-9dde-3400e5cab16e kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: 'Created pod: success-200-isvc-c88c1-predictor-5c979449cc-hqsj6' metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:54:56Z" name: success-200-isvc-c88c1-predictor-5c979449cc.18a90ab40c27ce97 namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: e441db31-86e0-4e74-95f9-183616388a94 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c88c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19069" uid: 7f14c41e-aea3-4e36-9f02-208b6bcd5d98 kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Scaled up replica set success-200-isvc-c88c1-predictor-5c979449cc from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:54:56Z" name: success-200-isvc-c88c1-predictor.18a90ab408f4f653 namespace: kserve-ci-e2e-test resourceVersion: "19073" uid: 1fbb3179-2cac-4aa2-b43b-07b1ff138a7b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c88c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19071" uid: 955b102d-acdb-4941-9bb9-cfbd2aec13d2 kind: Event lastTimestamp: "2026-04-23T16:55:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:55:26Z" name: success-200-isvc-c88c1-predictor.18a90ab787f1bff6 namespace: kserve-ci-e2e-test resourceVersion: "19379" uid: ba37812e-4c9d-4578-a60c-617178b380a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c88c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19071" uid: 955b102d-acdb-4941-9bb9-cfbd2aec13d2 kind: Event lastTimestamp: "2026-04-23T16:55:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:55:26Z" name: success-200-isvc-c88c1-predictor.18a90ab788529e5f namespace: kserve-ci-e2e-test resourceVersion: "19380" uid: 9916ae00-1c61-4b58-b42b-540bef9a3870 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:55:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c88c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19272" uid: 955b102d-acdb-4941-9bb9-cfbd2aec13d2 kind: Event lastTimestamp: "2026-04-23T16:56:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:11Z" name: success-200-isvc-c88c1-predictor.18a90abe861d4350 namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: d0d18b88-5696-4bf8-887b-e699243699d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T16:55:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c88c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19272" uid: 955b102d-acdb-4941-9bb9-cfbd2aec13d2 kind: Event lastTimestamp: "2026-04-23T16:56:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T16:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:56:11Z" name: success-200-isvc-c88c1-predictor.18a90abe867f87aa namespace: kserve-ci-e2e-test resourceVersion: "19924" uid: 8b95766d-f134-4332-86c7-763d576730ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c88c1 namespace: kserve-ci-e2e-test resourceVersion: "19052" uid: 05c99b9f-db47-43e5-a634-c3c8d07ff169 kind: Event lastTimestamp: "2026-04-23T16:55:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:55:48Z" name: success-200-isvc-c88c1.18a90ab408fca349 namespace: kserve-ci-e2e-test resourceVersion: "19691" uid: b2358f61-625c-47ed-b70c-a77fed905047 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:55:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c88c1 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 05c99b9f-db47-43e5-a634-c3c8d07ff169 kind: Event lastTimestamp: "2026-04-23T16:55:48Z" message: InferenceService [success-200-isvc-c88c1] is Ready metadata: creationTimestamp: "2026-04-23T16:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:55:48Z" name: success-200-isvc-c88c1.18a90ac02011c005 namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 246c4680-58fe-44cb-a10d-56308877a2b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T17:07:31.425345Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-390fe-857467bf76-rgb46 namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: 598bcef5-3c35-41c3-8ea3-8d94f8cb2782 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-390fe-857467bf76-rgb46 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T17:07:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T17:07:31Z" name: switch-graph-390fe-857467bf76-rgb46.18a90b63d7d8db69 namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 32121b9c-5bab-43d1-92a2-e2ff1106529e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-390fe-857467bf76-rgb46 namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: 598bcef5-3c35-41c3-8ea3-8d94f8cb2782 kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T17:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T17:07:31Z" name: switch-graph-390fe-857467bf76-rgb46.18a90b63f1490d03 namespace: kserve-ci-e2e-test resourceVersion: "25719" uid: 178117f5-3639-4043-9952-526bddcbc834 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390fe} kind: Pod name: switch-graph-390fe-857467bf76-rgb46 namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: 598bcef5-3c35-41c3-8ea3-8d94f8cb2782 kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T17:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:31Z" name: switch-graph-390fe-857467bf76-rgb46.18a90b63f278c0ac namespace: kserve-ci-e2e-test resourceVersion: "25721" uid: 6a57b694-abca-4af1-a618-63c7a311b509 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390fe} kind: Pod name: switch-graph-390fe-857467bf76-rgb46 namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: 598bcef5-3c35-41c3-8ea3-8d94f8cb2782 kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: 'Created container: switch-graph-390fe' metadata: creationTimestamp: "2026-04-23T17:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:31Z" name: switch-graph-390fe-857467bf76-rgb46.18a90b63f562b33a namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: 636d0d3a-f9a4-4f25-9e15-049badaa15a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390fe} kind: Pod name: switch-graph-390fe-857467bf76-rgb46 namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: 598bcef5-3c35-41c3-8ea3-8d94f8cb2782 kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: Started container switch-graph-390fe metadata: creationTimestamp: "2026-04-23T17:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:07:31Z" name: switch-graph-390fe-857467bf76-rgb46.18a90b63f5df4cca namespace: kserve-ci-e2e-test resourceVersion: "25726" uid: 01b16d8c-422d-42e6-97e5-8bd7ef288586 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390fe} kind: Pod name: switch-graph-390fe-857467bf76-rgb46 namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: 598bcef5-3c35-41c3-8ea3-8d94f8cb2782 kind: Event lastTimestamp: "2026-04-23T17:23:50Z" message: Stopping container switch-graph-390fe metadata: creationTimestamp: "2026-04-23T17:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:23:50Z" name: switch-graph-390fe-857467bf76-rgb46.18a90c47d606045c namespace: kserve-ci-e2e-test resourceVersion: "32900" uid: 96fa25bb-094e-40df-9ab3-8209dc0ade57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T17:23:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390fe} kind: Pod name: switch-graph-390fe-857467bf76-rgb46 namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: 598bcef5-3c35-41c3-8ea3-8d94f8cb2782 kind: Event lastTimestamp: "2026-04-23T17:24:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T17:23:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T17:24:13Z" name: switch-graph-390fe-857467bf76-rgb46.18a90c486ecd0a87 namespace: kserve-ci-e2e-test resourceVersion: "33131" uid: f174e479-38e5-4fa9-81d4-ce03b826a0da reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-390fe-857467bf76 namespace: kserve-ci-e2e-test resourceVersion: "25696" uid: 8d645d07-30b0-4af3-b248-a8b82dfa5210 kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: 'Created pod: switch-graph-390fe-857467bf76-rgb46' metadata: creationTimestamp: "2026-04-23T17:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:31Z" name: switch-graph-390fe-857467bf76.18a90b63d7508e25 namespace: kserve-ci-e2e-test resourceVersion: "25700" uid: 478cb8a9-5b9b-45b2-92e2-39b4842c0566 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-390fe namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: 2df9bca3-5064-4b4f-8fcb-78fded7eaea9 kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: Scaled up replica set switch-graph-390fe-857467bf76 from 0 to 1 metadata: creationTimestamp: "2026-04-23T17:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:31Z" name: switch-graph-390fe.18a90b63d67f7b3f namespace: kserve-ci-e2e-test resourceVersion: "25697" uid: 5470e92b-b713-432f-82df-82fe0db6921b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-390fe namespace: kserve-ci-e2e-test resourceVersion: "25712" uid: 52b50929-fb9b-4810-bd03-bd5d7bc700b3 kind: Event lastTimestamp: "2026-04-23T17:07:31Z" message: InferenceGraph [switch-graph-390fe] is Ready metadata: creationTimestamp: "2026-04-23T17:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T17:07:31Z" name: switch-graph-390fe.18a90b63db6e6a90 namespace: kserve-ci-e2e-test resourceVersion: "25714" uid: d03bd30b-d610-4a8c-97d2-2a92240f5608 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-390fe namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: 963f8010-c400-4a65-b1af-17cd9fc1b179 kind: Event lastTimestamp: "2026-04-23T17:07:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:46Z" name: switch-graph-390fe.18a90b6755640afc namespace: kserve-ci-e2e-test resourceVersion: "26125" uid: f3a88226-d2de-452a-979d-8f5e38785141 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:07:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-390fe namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: 963f8010-c400-4a65-b1af-17cd9fc1b179 kind: Event lastTimestamp: "2026-04-23T17:07:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T17:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:07:46Z" name: switch-graph-390fe.18a90b6755c4e9df namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 6abd1efa-1ea1-424e-a15f-4bc0a08f1fa7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-390fe namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 963f8010-c400-4a65-b1af-17cd9fc1b179 kind: Event lastTimestamp: "2026-04-23T17:08:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:01Z" name: switch-graph-390fe.18a90b6ad47bd5e1 namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 7a125cf9-0275-4c05-86cb-0483e63ff70d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T17:08:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-390fe namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 963f8010-c400-4a65-b1af-17cd9fc1b179 kind: Event lastTimestamp: "2026-04-23T17:08:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T17:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T17:08:01Z" name: switch-graph-390fe.18a90b6ad4dee9d9 namespace: kserve-ci-e2e-test resourceVersion: "26266" uid: 2e813b30-85a8-4b99-85e8-986974bffb98 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:45:21.071569Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5bdc4-7467b6c785-7tk6l namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 49efd145-4d17-4fa2-9101-2ba8c03d6df6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-5bdc4-7467b6c785-7tk6l to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:45:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:45:21Z" name: switch-graph-5bdc4-7467b6c785-7tk6l.18a90a2e18953a76 namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 0305fcae-0222-4ace-a281-8609fa53431d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5bdc4-7467b6c785-7tk6l namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 49efd145-4d17-4fa2-9101-2ba8c03d6df6 kind: Event lastTimestamp: "2026-04-23T16:45:21Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:45:21Z" name: switch-graph-5bdc4-7467b6c785-7tk6l.18a90a2e310443ca namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: 8976c55f-8927-4a26-9344-fee6c3289453 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bdc4} kind: Pod name: switch-graph-5bdc4-7467b6c785-7tk6l namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 49efd145-4d17-4fa2-9101-2ba8c03d6df6 kind: Event lastTimestamp: "2026-04-23T16:45:21Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" metadata: creationTimestamp: "2026-04-23T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:21Z" name: switch-graph-5bdc4-7467b6c785-7tk6l.18a90a2e324aa0e4 namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 5b7f921e-9649-401f-8383-9b9afc847f33 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bdc4} kind: Pod name: switch-graph-5bdc4-7467b6c785-7tk6l namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 49efd145-4d17-4fa2-9101-2ba8c03d6df6 kind: Event lastTimestamp: "2026-04-23T16:45:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" in 2.319s (2.319s including waiting). Image size: 216232504 bytes.' metadata: creationTimestamp: "2026-04-23T16:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:23Z" name: switch-graph-5bdc4-7467b6c785-7tk6l.18a90a2ebc85e448 namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 54a0a545-2b6f-45d1-9bd9-a2ea75de3b6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bdc4} kind: Pod name: switch-graph-5bdc4-7467b6c785-7tk6l namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 49efd145-4d17-4fa2-9101-2ba8c03d6df6 kind: Event lastTimestamp: "2026-04-23T16:45:23Z" message: 'Created container: switch-graph-5bdc4' metadata: creationTimestamp: "2026-04-23T16:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:23Z" name: switch-graph-5bdc4-7467b6c785-7tk6l.18a90a2ebf47ce8c namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: 9610d7ed-df75-4aef-9870-29e92958203e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bdc4} kind: Pod name: switch-graph-5bdc4-7467b6c785-7tk6l namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 49efd145-4d17-4fa2-9101-2ba8c03d6df6 kind: Event lastTimestamp: "2026-04-23T16:45:23Z" message: Started container switch-graph-5bdc4 metadata: creationTimestamp: "2026-04-23T16:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:23Z" name: switch-graph-5bdc4-7467b6c785-7tk6l.18a90a2ebfb7229f namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 4da50e92-a235-45f1-8a20-47e43c6002b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bdc4} kind: Pod name: switch-graph-5bdc4-7467b6c785-7tk6l namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 49efd145-4d17-4fa2-9101-2ba8c03d6df6 kind: Event lastTimestamp: "2026-04-23T16:45:31Z" message: Stopping container switch-graph-5bdc4 metadata: creationTimestamp: "2026-04-23T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:45:31Z" name: switch-graph-5bdc4-7467b6c785-7tk6l.18a90a307636d93d namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: d3c7aea3-2ad3-43e4-a15b-7337fcc3d2c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bdc4} kind: Pod name: switch-graph-5bdc4-7467b6c785-7tk6l namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 49efd145-4d17-4fa2-9101-2ba8c03d6df6 kind: Event lastTimestamp: "2026-04-23T16:46:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T16:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:00Z" name: switch-graph-5bdc4-7467b6c785-7tk6l.18a90a3180ccee2a namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: e4923586-2828-4449-a024-321ff2491700 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-5bdc4-7467b6c785 namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: e73213fe-24c8-44ea-aa14-25b9902d36dc kind: Event lastTimestamp: "2026-04-23T16:45:21Z" message: 'Created pod: switch-graph-5bdc4-7467b6c785-7tk6l' metadata: creationTimestamp: "2026-04-23T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:21Z" name: switch-graph-5bdc4-7467b6c785.18a90a2e17b4cc82 namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: 7c923da7-f045-48f2-b446-5f583073af8c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: ce13523e-8234-43d1-a210-f4a73439ae30 kind: Event lastTimestamp: "2026-04-23T16:45:21Z" message: Scaled up replica set switch-graph-5bdc4-7467b6c785 from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:45:21Z" name: switch-graph-5bdc4.18a90a2e16e574e7 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: bbc1ade3-1477-4f5c-9cbd-1f5a58581efc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: d801e9a6-f4e1-415a-beda-d16245359f32 kind: Event lastTimestamp: "2026-04-23T16:45:21Z" message: InferenceGraph [switch-graph-5bdc4] is Ready metadata: creationTimestamp: "2026-04-23T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:21Z" name: switch-graph-5bdc4.18a90a2e1d203ffb namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: d082e97a-e3c6-4bb6-a634-4d93396fd426 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: d801e9a6-f4e1-415a-beda-d16245359f32 kind: Event lastTimestamp: "2026-04-23T16:45:21Z" message: 'Failed to update status for InferenceGraph "switch-graph-5bdc4": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-5bdc4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:21Z" name: switch-graph-5bdc4.18a90a2e1f4a075d namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 13bef2c5-fd3a-4fe2-9e5d-18f08fac2f9f reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-5bdc4 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: d801e9a6-f4e1-415a-beda-d16245359f32 kind: Event lastTimestamp: "2026-04-23T16:45:21Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-5bdc4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:45:21Z" name: switch-graph-5bdc4.18a90a2e1f4a4c09 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 339d8fcb-17bd-4c34-9518-bd60df0390b9 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T16:46:41.483906Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-be540-765869d5bd-xhdh2 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 9b5e2a72-ff54-4967-8fee-148b24ebf2be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-be540-765869d5bd-xhdh2 to ip-10-0-141-189.ec2.internal metadata: creationTimestamp: "2026-04-23T16:46:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T16:46:41Z" name: switch-graph-be540-765869d5bd-xhdh2.18a90a40d188244d namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 2e84732f-59a3-496d-ad7b-7520017c0d2f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f4fb6c578-5ctbf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:41Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-be540-765869d5bd-xhdh2 namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: 9b5e2a72-ff54-4967-8fee-148b24ebf2be kind: Event lastTimestamp: "2026-04-23T16:46:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-be540-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T16:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:41Z" name: switch-graph-be540-765869d5bd-xhdh2.18a90a40de16e9d2 namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: ebae500f-9a45-4042-9d03-13f0f9eb0568 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:42Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-be540-765869d5bd-xhdh2 namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: 9b5e2a72-ff54-4967-8fee-148b24ebf2be kind: Event lastTimestamp: "2026-04-23T16:46:42Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T16:46:42Z" name: switch-graph-be540-765869d5bd-xhdh2.18a90a410e57b790 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: e5fc07db-bf24-4bac-bf10-96d95326b453 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be540} kind: Pod name: switch-graph-be540-765869d5bd-xhdh2 namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: 9b5e2a72-ff54-4967-8fee-148b24ebf2be kind: Event lastTimestamp: "2026-04-23T16:46:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:66a46d5a6623872616c85560bba9e0c6b4eaf29ca86ba636e84212e5017220b2" already present on machine metadata: creationTimestamp: "2026-04-23T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:42Z" name: switch-graph-be540-765869d5bd-xhdh2.18a90a410f91f55a namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: c07e9e73-542e-4305-8b6d-97f356f39514 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be540} kind: Pod name: switch-graph-be540-765869d5bd-xhdh2 namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: 9b5e2a72-ff54-4967-8fee-148b24ebf2be kind: Event lastTimestamp: "2026-04-23T16:46:42Z" message: 'Created container: switch-graph-be540' metadata: creationTimestamp: "2026-04-23T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:42Z" name: switch-graph-be540-765869d5bd-xhdh2.18a90a4112bb40e3 namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 742d56fd-7cd8-42c3-ae67-0cce3aecc08e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be540} kind: Pod name: switch-graph-be540-765869d5bd-xhdh2 namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: 9b5e2a72-ff54-4967-8fee-148b24ebf2be kind: Event lastTimestamp: "2026-04-23T16:46:42Z" message: Started container switch-graph-be540 metadata: creationTimestamp: "2026-04-23T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:46:42Z" name: switch-graph-be540-765869d5bd-xhdh2.18a90a41132911fe namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 44c03b40-e277-4687-bca2-51900e897889 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:54:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be540} kind: Pod name: switch-graph-be540-765869d5bd-xhdh2 namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: 9b5e2a72-ff54-4967-8fee-148b24ebf2be kind: Event lastTimestamp: "2026-04-23T16:54:56Z" message: Stopping container switch-graph-be540 metadata: creationTimestamp: "2026-04-23T16:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:54:56Z" name: switch-graph-be540-765869d5bd-xhdh2.18a90ab40310a11d namespace: kserve-ci-e2e-test resourceVersion: "19049" uid: ab0547fc-02d3-48b7-b101-be8408f8e2c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T16:54:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-be540} kind: Pod name: switch-graph-be540-765869d5bd-xhdh2 namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: 9b5e2a72-ff54-4967-8fee-148b24ebf2be kind: Event lastTimestamp: "2026-04-23T16:55:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T16:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T16:55:23Z" name: switch-graph-be540-765869d5bd-xhdh2.18a90ab4a370e7b0 namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: 04826671-ede2-4d8c-b51e-f0de8e21daec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-189.ec2.internal source: component: kubelet host: ip-10-0-141-189.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-be540-765869d5bd namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 3a12d804-3509-4bde-8636-727a267aaf5a kind: Event lastTimestamp: "2026-04-23T16:46:41Z" message: 'Created pod: switch-graph-be540-765869d5bd-xhdh2' metadata: creationTimestamp: "2026-04-23T16:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:41Z" name: switch-graph-be540-765869d5bd.18a90a40d1204c6e namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: e98bd23c-3fcc-4f74-82d9-76b0b19f763d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-be540 namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: 87d65646-0ed0-4a1d-a1aa-818fa3c7a119 kind: Event lastTimestamp: "2026-04-23T16:46:41Z" message: Scaled up replica set switch-graph-be540-765869d5bd from 0 to 1 metadata: creationTimestamp: "2026-04-23T16:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:46:41Z" name: switch-graph-be540.18a90a40d015e206 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 2a06aaad-cafb-4bd8-a6e9-f1e28d653be8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T16:46:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-be540 namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: 1f67bcef-d762-461a-b207-9ae7f8f28962 kind: Event lastTimestamp: "2026-04-23T16:46:41Z" message: InferenceGraph [switch-graph-be540] is Ready metadata: creationTimestamp: "2026-04-23T16:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T16:46:41Z" name: switch-graph-be540.18a90a40d5b308b4 namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: b65ec765-b7cc-4bed-8c56-86ced6013ab7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:46:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-be540 namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 81e23b55-e77c-488c-a26b-c688c7e37d71 kind: Event lastTimestamp: "2026-04-23T16:47:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:47:11Z" name: switch-graph-be540.18a90a444ea44a54 namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: b46172f4-ce88-4e07-b767-848253dc7990 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T16:46:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-be540 namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 81e23b55-e77c-488c-a26b-c688c7e37d71 kind: Event lastTimestamp: "2026-04-23T16:47:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T16:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T16:47:11Z" name: switch-graph-be540.18a90a444f03fafc namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: 9ed1a107-17ff-498d-a510-3693514592b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33136"